[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
@@ -77,21 +77,3 @@ class ECConnectorFactory:
# Register various connectors here.
# The registration should not be done in each individual file, as we want to
# only load the files corresponding to the current connector.
ECConnectorFactory.register_connector(
"ECExampleConnector",
"vllm.distributed.ec_transfer.ec_connector.example_connector",
"ECExampleConnector",
)
ECConnectorFactory.register_connector(
"ECCPUConnector",
"vllm.distributed.ec_transfer.ec_connector.cpu.connector",
"ECCPUConnector",
)
ECConnectorFactory.register_connector(
"ECMooncakeConnector",
"vllm.distributed.ec_transfer.ec_connector.mooncake_ec_connector",
"ECMooncakeConnector",
)
@@ -150,48 +150,12 @@ class KVConnectorFactory:
# The registration should not be done in each individual file, as we want to
# only load the files corresponding to the current connector.
KVConnectorFactory.register_connector(
"ExampleHiddenStatesConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector",
"ExampleHiddenStatesConnector",
)
KVConnectorFactory.register_connector(
"NixlConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
"NixlConnector",
)
KVConnectorFactory.register_connector(
"NixlPullConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
"NixlPullConnector",
)
KVConnectorFactory.register_connector(
"NixlPushConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
"NixlPushConnector",
)
KVConnectorFactory.register_connector(
"MultiConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.multi_connector",
"MultiConnector",
)
KVConnectorFactory.register_connector(
"HiSparseConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector",
"HiSparseConnector",
)
KVConnectorFactory.register_connector(
"MoRIIOConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector",
"MoRIIOConnector",
)
KVConnectorFactory.register_connector(
"OffloadingConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector",
@@ -203,14 +167,3 @@ KVConnectorFactory.register_connector(
"vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector",
"DecodeBenchConnector",
)
KVConnectorFactory.register_connector(
"SimpleCPUOffloadConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector",
"SimpleCPUOffloadConnector",
)
KVConnectorFactory.register_connector(
"HF3FSKVConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_connector",
"HF3FSKVConnector",
)
@@ -1,302 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import multiprocessing
import os
import threading
from functools import wraps
from pathlib import Path
import torch
import torch.utils.cpp_extension
from torch.utils.cpp_extension import load
root = Path(__file__).parent.resolve()
cuda_include_path = os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")
hf3fs_utils = load(
name="hf3fs_utils",
sources=[f"{root}/utils/hf3fs_utils.cpp"],
extra_include_paths=[cuda_include_path],
)
logger = logging.getLogger(__name__)
HF3FS_AVAILABLE = True
try:
from hf3fs_fuse.io import (
deregister_fd,
extract_mount_point,
make_ioring,
make_iovec,
register_fd,
)
except ImportError:
HF3FS_AVAILABLE = False
def rsynchronized():
def _decorator(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
with self.rlock:
return func(self, *args, **kwargs)
return wrapper
return _decorator
def wsynchronized():
def _decorator(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
with self.wlock:
return func(self, *args, **kwargs)
return wrapper
return _decorator
class Hf3fsClient:
def __init__(self, path: str, size: int, bytes_per_page: int, entries: int):
"""Initialize the HF3FS client with hf3fs_fuse.
Args:
path: Path to the file used for storage
size: Total size of the storage file in bytes
bytes_per_page: Size of each page in bytes
entries: Maximum number of concurrent operations
"""
if not HF3FS_AVAILABLE:
raise ImportError(
"hf3fs_fuse.io is not available. Please install the hf3fs_fuse package."
)
self.path = path
self.size = size
self.bytes_per_page = bytes_per_page
self.entries = entries
self._closed = False
self.file = None
self.shm_r = None
self.shm_w = None
self.ior_r = None
self.ior_w = None
self.iov_r = None
self.iov_w = None
try:
# Create the file if it doesn't exist and set its size
self.file = os.open(self.path, os.O_RDWR | os.O_CREAT)
os.ftruncate(self.file, size)
register_fd(self.file)
self.hf3fs_mount_point = extract_mount_point(path)
self.bs = self.bytes_per_page
self.shm_r = multiprocessing.shared_memory.SharedMemory(
size=self.bs * self.entries, create=True
)
self.shm_w = multiprocessing.shared_memory.SharedMemory(
size=self.bs * self.entries, create=True
)
self.shm_r_tensor = torch.frombuffer(self.shm_r.buf, dtype=torch.uint8)
self.shm_w_tensor = torch.frombuffer(self.shm_w.buf, dtype=torch.uint8)
numel = self.bs * self.entries
self.r_pinned = torch.empty(
numel,
dtype=torch.uint8,
device="cpu",
pin_memory=True,
)
self.w_pinned = torch.empty(
numel,
dtype=torch.uint8,
device="cpu",
pin_memory=True,
)
self.numa = -1
self.ior_r = make_ioring(
self.hf3fs_mount_point,
self.entries,
for_read=True,
timeout=1,
numa=self.numa,
)
self.ior_w = make_ioring(
self.hf3fs_mount_point,
self.entries,
for_read=False,
timeout=1,
numa=self.numa,
)
self.iov_r = make_iovec(self.shm_r, self.hf3fs_mount_point)
self.iov_w = make_iovec(self.shm_w, self.hf3fs_mount_point)
self.shm_r.unlink()
self.shm_w.unlink()
self.rlock = threading.RLock()
self.wlock = threading.RLock()
self.stream = torch.cuda.Stream()
self.stream_ptr_int = self.stream.cuda_stream
except Exception:
self._release_resources()
raise
logger.debug(
"Initialized HF3FS client with file: %s, size: %s bytes", path, size
)
def _release_resources(self) -> None:
"""Release all acquired resources safely."""
# iov must be released before ioring and shm
for attr in ("iov_r", "iov_w", "ior_r", "ior_w"):
obj = getattr(self, attr, None)
if obj is not None:
del obj
setattr(self, attr, None)
for attr in ("shm_r", "shm_w"):
shm = getattr(self, attr, None)
if shm is not None:
try:
shm.close()
except Exception as e:
logger.warning("Failed to close %s: %s", attr, e)
setattr(self, attr, None)
if self.file is not None:
try:
deregister_fd(self.file)
except Exception as e:
logger.warning("deregister_fd failed: %s", e)
try:
os.close(self.file)
except OSError as e:
logger.warning("os.close failed: %s", e)
self.file = None
@rsynchronized()
def batch_read(self, offsets: list[int], tensors: list[torch.Tensor]) -> list[int]:
"""Read data from the file at specified offsets into tensors.
Args:
offsets: List of byte offsets to read from
tensors: List of tensors to read data into
Returns:
List of operation results (0 for success, non-zero for error)
"""
self.check(offsets, tensors)
assert self.ior_r is not None
assert self.iov_r is not None
# prepare
current = 0
for offset, tensor in zip(offsets, tensors):
size = tensor.numel() * tensor.itemsize
self.ior_r.prepare(
self.iov_r[current : current + size], True, self.file, offset
)
current += size
# submit
ionum = len(offsets)
resv = self.ior_r.submit().wait(min_results=ionum)
# results
with torch.cuda.stream(self.stream):
hf3fs_utils.read_shm(
self.shm_r_tensor, self.r_pinned, tensors, self.stream_ptr_int
)
results = [res.result for res in resv]
return results
@wsynchronized()
def batch_write(
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
) -> list[int]:
"""Write data from tensors to the file at specified offsets.
Args:
offsets: List of byte offsets to write to
tensors: List of tensors containing data to write
event: CUDA event to synchronize on before reading the tensors
Returns:
List of operation results (0 for success, non-zero for error)
"""
self.check(offsets, tensors)
assert self.ior_w is not None
assert self.iov_w is not None
# prepare
with torch.cuda.stream(self.stream):
self.stream.wait_event(event)
hf3fs_utils.write_shm(
tensors, self.shm_w_tensor, self.w_pinned, self.stream_ptr_int
)
current = 0
for offset, tensor in zip(offsets, tensors):
size = tensor.numel() * tensor.itemsize
self.ior_w.prepare(
self.iov_w[current : current + size], False, self.file, offset
)
current += size
# submit
ionum = len(offsets)
resv = self.ior_w.submit().wait(min_results=ionum)
# results
results = [res.result for res in resv]
return results
def check(self, offsets: list[int], tensors: list[torch.Tensor]) -> None:
sizes = [t.numel() * t.itemsize for t in tensors]
if any(
[
len(offsets) > self.entries,
len(offsets) != len(sizes),
any(
offset < 0 or offset + size > self.size
for offset, size in zip(offsets, sizes)
),
any(size > self.bytes_per_page for size in sizes),
]
):
self.close()
raise ValueError("Hf3fsClient.check Failed")
def get_size(self) -> int:
"""Get the total size of the storage file.
Returns:
Size of the file in bytes
"""
return self.size
def close(self) -> None:
"""Close the client and clean up resources."""
if self._closed:
return
self._closed = True
self._release_resources()
def flush(self) -> None:
"""Flush any pending writes to disk."""
if not self._closed and self.file is not None:
os.fsync(self.file)
File diff suppressed because it is too large Load Diff
@@ -1,532 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HF3FS Metadata Server with key-based organization."""
import argparse
import logging
import threading
from abc import ABC, abstractmethod
from dataclasses import dataclass
try:
import orjson
HAS_ORJSON = True
except ImportError:
import json as orjson # type: ignore
HAS_ORJSON = False
import requests
from fastapi import FastAPI, HTTPException, Request, Response
from fastapi.responses import ORJSONResponse
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)
@dataclass
class RankFileMetadata:
"""Manages file page allocation for a single rank."""
rank_id: int
num_pages: int
free_pages: list[int]
def allocate_pages(self, num_pages: int) -> list[int]:
"""Allocate specified number of free pages."""
if len(self.free_pages) < num_pages:
return []
allocated = self.free_pages[:num_pages]
self.free_pages = self.free_pages[num_pages:]
return allocated
def release_pages(self, page_indices: list[int]) -> None:
"""Release pages back to free pool."""
for page_idx in page_indices:
if page_idx not in self.free_pages:
self.free_pages.append(page_idx)
def get_free_page_count(self) -> int:
"""Get current number of free pages."""
return len(self.free_pages)
@dataclass
class KeyMetadata:
"""Manages metadata for a single key across multiple ranks."""
key: str
rank_to_page: dict[int, int] # rank -> allocated page index
tp_world_size: int
def add_rank_page(self, rank: int, page_index: int) -> None:
"""Add page allocation for a specific rank."""
self.rank_to_page[rank] = page_index
def get_all_pages(self) -> list[tuple[int, int]]:
"""Get all (rank, page) pairs for this key."""
return [(rank, page) for rank, page in self.rank_to_page.items()]
def get_rank_page(self, rank: int) -> int | None:
"""Get page index for a specific rank."""
return self.rank_to_page.get(rank)
def is_complete(self) -> bool:
"""Check if all ranks in the TP world have allocated pages."""
return len(self.rank_to_page) == self.tp_world_size
class GlobalMetadataState:
"""Manages global metadata state across all ranks and keys."""
def __init__(self):
self.global_lock = threading.RLock()
self.rank_metadata: dict[int, RankFileMetadata] = {}
self.key_metadata: dict[str, KeyMetadata] = {}
def clear(self) -> None:
"""Clear all metadata state."""
with self.global_lock:
self.rank_metadata.clear()
self.key_metadata.clear()
logger.info("Cleared all metadata state")
def initialize_rank(self, rank: int, num_pages: int) -> None:
"""Initialize a new rank with specified number of pages."""
with self.global_lock:
if rank not in self.rank_metadata:
self.rank_metadata[rank] = RankFileMetadata(
rank, num_pages, list(range(num_pages))
)
logger.info("Initialized rank %s with %s pages", rank, num_pages)
def allocate_pages_for_keys(
self, rank: int, keys: list[tuple[str, str]]
) -> dict[str, int]:
"""Allocate one page for each key on the specified rank.
Args:
rank: Rank ID to allocate pages on
keys: List of keys to allocate pages for
Returns:
Dictionary mapping key -> allocated page index
"""
with self.global_lock:
if rank not in self.rank_metadata:
raise ValueError(f"Rank {rank} not initialized")
# Batch allocate pages for all keys
num_pages_needed = len(keys)
allocated_pages = self.rank_metadata[rank].allocate_pages(num_pages_needed)
if len(allocated_pages) < num_pages_needed:
logger.warning(
"Rank %s only allocated %s pages for %s keys",
rank,
len(allocated_pages),
num_pages_needed,
)
allocation_results = {}
for i, (key, prefix_key) in enumerate(keys):
if key in self.key_metadata:
key_meta = self.key_metadata[key]
if key_meta.is_complete() and rank in key_meta.rank_to_page:
# key is already fully written, reuse the existing page
# and release the allocated pages back to the free pool.
if i < len(allocated_pages):
self.rank_metadata[rank].release_pages([allocated_pages[i]])
allocation_results[key] = key_meta.rank_to_page[rank]
continue
if i < len(allocated_pages):
allocation_results[key] = allocated_pages[i]
else:
allocation_results[key] = -1 # No pages available
return allocation_results
def confirm_write_for_keys(
self,
rank: int,
key_confirmations: list[tuple[str, int]],
pages_to_release: list[int] | None = None,
) -> None:
"""Confirm write operations for keys and update metadata.
Args:
rank: Rank ID that confirmed the writes
key_confirmations: List of (key, page_index) tuples
pages_to_release: List of page indices to release back to free pool
"""
with self.global_lock:
# Confirm successful writes
for key, page_index in key_confirmations:
if key not in self.key_metadata:
# Need to determine tp_world_size from rank_metadata
tp_world_size = len(self.rank_metadata)
self.key_metadata[key] = KeyMetadata(key, {}, tp_world_size)
# Add confirmed page to key metadata
self.key_metadata[key].add_rank_page(rank, page_index)
# Release specified pages back to free pool
if pages_to_release:
self.rank_metadata[rank].release_pages(pages_to_release)
logger.debug(
"Released %s pages on rank %s: %s",
len(pages_to_release),
rank,
pages_to_release,
)
def batch_key_exists(self, keys: list[str]) -> list[bool]:
"""Check if keys exist in metadata and all ranks have confirmed writes.
Args:
keys: List of keys to check
Returns:
List of boolean values indicating key existence and completion
"""
with self.global_lock:
results = []
for key in keys:
if key not in self.key_metadata:
results.append(False)
else:
# Check if all ranks in the TP world have confirmed writes
key_meta = self.key_metadata[key]
results.append(key_meta.is_complete())
return results
def get_key_locations(self, rank: int, keys: list[str]) -> list[int | None]:
"""Get page indices for keys on a specific rank.
Args:
rank: Rank ID to query
keys: List of keys to look up
Returns:
List of page indices in the same order as input keys (None if key not found)
"""
with self.global_lock:
if rank not in self.rank_metadata:
raise ValueError(f"Rank {rank} not initialized")
results = []
for key in keys:
if key in self.key_metadata:
key_meta = self.key_metadata[key]
if key_meta.is_complete():
page_index = key_meta.get_rank_page(rank)
else:
page_index = None
results.append(page_index)
else:
results.append(None)
return results
class Hf3fsMetadataServer:
"""HF3FS Metadata Server with improved key-based organization."""
def __init__(self, persistence_path: str | None = None, save_interval: int = 60):
self.state = GlobalMetadataState()
if HAS_ORJSON:
self.app = FastAPI(default_response_class=ORJSONResponse)
else:
self.app = FastAPI()
self._setup_routes()
async def _read_json(self, request: Request) -> dict:
"""Parse request JSON using orjson if available."""
body = await request.body()
return orjson.loads(body)
def _json_response(self, content: dict):
"""Return ORJSONResponse when available to bypass jsonable_encoder."""
if HAS_ORJSON:
return ORJSONResponse(content)
else:
return content
def _setup_routes(self):
"""Setup FastAPI routes for new API design."""
self.app.post("/rank/{rank}/initialize")(self.initialize_rank)
self.app.post("/keys/batch_allocate")(self.batch_allocate_pages_for_keys)
self.app.post("/keys/confirm_write")(self.confirm_write_for_keys)
self.app.post("/keys/batch_exists")(self.batch_key_exists)
self.app.post("/keys/get_locations")(self.get_key_locations)
self.app.post("/clear")(self.clear)
async def initialize_rank(self, rank: int, request: Request):
"""Initialize a rank with specified number of pages."""
data = await self._read_json(request)
role = data.get("role", "worker")
num_pages = data.get("num_pages", 0)
if role == "scheduler":
return self._json_response(
{"message": "Scheduler role does not require initialization"}
)
if role == "worker" and num_pages > 0:
self.state.initialize_rank(rank, num_pages)
return self._json_response(
{"message": f"Rank {rank} initialized with {num_pages} pages"}
)
else:
raise HTTPException(
status_code=400, detail="Invalid initialization parameters"
)
async def batch_allocate_pages_for_keys(self, request: Request):
"""Allocate one page for each key on a specific rank."""
data = await self._read_json(request)
rank = data.get("rank")
keys = data.get("keys", [])
# Validate input format
if rank is None or not isinstance(keys, list):
raise HTTPException(
status_code=400, detail="Invalid request format: need 'rank' and 'keys'"
)
try:
# Perform allocation
results = self.state.allocate_pages_for_keys(rank, keys)
# Convert results to response format
response = {"rank": rank, "results": list(results.items())}
return self._json_response(response)
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Allocation failed: {str(e)}"
) from e
async def confirm_write_for_keys(self, request: Request):
"""Confirm write operations for keys."""
data = await self._read_json(request)
rank = data.get("rank")
confirmations = data.get("confirmations", [])
pages_to_release = data.get("pages_to_release", [])
# Validate input format
if rank is None or not isinstance(confirmations, list):
raise HTTPException(
status_code=400,
detail="Invalid request format: need 'rank' and 'confirmations'",
)
try:
self.state.confirm_write_for_keys(rank, confirmations, pages_to_release)
return Response(status_code=204)
except Exception as e:
logger.error("Confirm write for keys failed: %s", e)
raise HTTPException(
status_code=500, detail=f"Confirmation failed: {str(e)}"
) from e
async def batch_key_exists(self, request: Request):
"""Check if multiple keys exist in metadata."""
data = await self._read_json(request)
keys = data.get("keys", [])
if not isinstance(keys, list):
raise HTTPException(status_code=400, detail="Invalid keys format")
try:
exists_results = self.state.batch_key_exists(keys)
return self._json_response({"exists": exists_results})
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Key existence check failed: {str(e)}"
) from e
async def get_key_locations(self, request: Request):
"""Get page indices for keys on a specific rank."""
data = await self._read_json(request)
rank = data.get("rank")
keys = data.get("keys", [])
# Validate input format
if rank is None or not isinstance(keys, list):
raise HTTPException(
status_code=400, detail="Invalid request format: need 'rank' and 'keys'"
)
try:
# Get key locations
locations = self.state.get_key_locations(rank, keys)
return self._json_response({"locations": locations})
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Failed to get key locations: {str(e)}"
) from e
async def clear(self, request: Request):
"""Clear the metadata server."""
self.state.clear()
return Response(status_code=204)
def run(self, host: str = "0.0.0.0", port: int = 18000):
"""Run the metadata server."""
import uvicorn
logger.info("Starting improved metadata server on http://%s:%s", host, port)
uvicorn.run(self.app, host=host, port=port)
# --- Client implementation ---
class Hf3fsMetadataInterface(ABC):
"""Interface for HF3FS metadata operations."""
@abstractmethod
def initialize(self, rank: int, num_pages: int = 0, role: str = "worker") -> None:
"""Initialize the metadata service with specified number of pages."""
pass
@abstractmethod
def allocate_pages_for_keys(
self, rank: int, keys: list[tuple[str, str]]
) -> list[tuple[str, int]]:
"""Allocate one page for each key on the specified rank."""
pass
@abstractmethod
def confirm_write_for_keys(
self,
rank: int,
key_confirmations: list[tuple[str, int]],
pages_to_release: list[int] | None = None,
) -> None:
"""Confirm write operations for keys and optionally release pages."""
pass
@abstractmethod
def batch_key_exists(self, keys: list[str]) -> list[bool]:
"""Check if keys exist and are complete across all ranks."""
pass
@abstractmethod
def get_key_locations(self, rank: int, keys: list[str]) -> list[int]:
"""Get page indices for keys on a specific rank."""
pass
class Hf3fsGlobalMetadataClient(Hf3fsMetadataInterface):
"""Global HTTP metadata client for HF3FS."""
def __init__(self, base_url: str = "http://localhost:18000", max_retries: int = 3):
self.base_url = base_url.rstrip("/")
self._session = requests.Session()
retry_strategy = Retry(
total=max_retries,
backoff_factor=0.3,
status_forcelist=[500, 502, 503, 504],
allowed_methods=["GET", "POST"],
)
adapter = HTTPAdapter(max_retries=retry_strategy)
self._session.mount("http://", adapter)
def _post(self, endpoint: str, json_data: dict) -> dict:
"""Make POST request to metadata server."""
try:
url = f"{self.base_url}/{endpoint}"
headers = {"Content-Type": "application/json"}
if HAS_ORJSON:
payload = orjson.dumps(json_data)
else:
import json
payload = json.dumps(json_data).encode("utf-8")
response = self._session.post(url, data=payload, headers=headers)
response.raise_for_status()
if response.status_code == 204 or not response.content:
return {}
if HAS_ORJSON:
return orjson.loads(response.content)
else:
return response.json()
except requests.exceptions.RequestException as e:
logger.error("Failed to POST to %s after retries: %s", endpoint, e)
raise RuntimeError(f"Failed to connect to metadata server: {e}") from e
def initialize(self, rank: int, num_pages: int = 0, role: str = "worker") -> None:
"""Initialize a rank with specified number of pages."""
self._post(f"rank/{rank}/initialize", {"num_pages": num_pages, "role": role})
def allocate_pages_for_keys(
self, rank: int, keys: list[tuple[str, str]]
) -> list[tuple[str, int]]:
"""Allocate pages for keys on the specified rank."""
response = self._post("keys/batch_allocate", {"rank": rank, "keys": keys})
# Convert response to expected format
return response.get("results", {})
def confirm_write_for_keys(
self,
rank: int,
key_confirmations: list[tuple[str, int]],
pages_to_release: list[int] | None = None,
) -> None:
"""Confirm write operations for keys and optionally release pages."""
payload = {
"rank": rank,
"confirmations": key_confirmations,
"pages_to_release": pages_to_release or [],
}
self._post("keys/confirm_write", payload)
def batch_key_exists(self, keys: list[str]) -> list[bool]:
"""Check if keys exist and are complete across all ranks."""
response = self._post("keys/batch_exists", {"keys": keys})
return response.get("exists", [])
def get_key_locations(self, rank: int, keys: list[str]) -> list[int]:
"""Get page indices for keys on a specific rank."""
response = self._post("keys/get_locations", {"rank": rank, "keys": keys})
return response.get("locations", [])
def run_metadata_server(
host: str = "0.0.0.0",
port: int = 18000,
):
"""Run the improved HF3FS metadata server."""
server = Hf3fsMetadataServer()
server.run(host=host, port=port)
# --- Main Execution ---
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Improved HF3FS Metadata Server")
parser.add_argument(
"--host", type=str, default="0.0.0.0", help="Host to bind the server to."
)
parser.add_argument(
"--port", type=int, default=18000, help="Port to run the server on."
)
args = parser.parse_args()
run_metadata_server(args.host, args.port)
@@ -1,139 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from dataclasses import dataclass, field
from typing import Optional
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.v1.request import Request
class AtomicCounter:
"""Thread-safe atomic counter for round-robin operations."""
def __init__(self, n: int):
assert n > 0, "Counter size must be positive"
self._n = n
self._value = 0
self._lock = threading.Lock()
def next(self) -> int:
"""Get next value in round-robin fashion."""
with self._lock:
current = self._value
self._value = (current + 1) % self._n
return current
@dataclass
class LoadBlockInfo:
"""Operation for loading blocks from external storage."""
num_computed_blocks: int
num_blocks_to_load: int
need_fetch_block_ids: list[int]
@dataclass
class SaveBlockInfo:
"""Operation for saving blocks to external storage."""
skip_leading_blocks: int
@dataclass
class RequestSchedulingState:
"""Unified request scheduling state management."""
request_id: str
request: Request | None = None
# Token and block tracking
token_ids: list[int] = field(default_factory=list)
allocated_block_ids: list[int] = field(default_factory=list)
num_saved_blocks: int = 0
# Load operation info
load_op: LoadBlockInfo | None = None
# Scheduling phase
phase: str = "NEW" # NEW -> WAITING_TO_LOAD -> ACTIVE -> FINISHED
def needs_loading(self) -> bool:
"""Check if request needs loading."""
return self.load_op is not None and self.load_op.num_blocks_to_load > 0
def is_ready_to_load(self) -> bool:
"""Check if request is ready for loading."""
return self.phase == "WAITING_TO_LOAD" and self.needs_loading()
def update_tokens_and_blocks(self, new_token_ids: list[int], new_block_ids) -> None:
"""Update with new tokens and blocks."""
if new_token_ids:
self.token_ids.extend(new_token_ids)
if new_block_ids is not None:
normalized_block_ids = self._normalize_block_ids(new_block_ids)
self.allocated_block_ids.extend(normalized_block_ids)
def _normalize_block_ids(self, block_ids) -> list[int]:
"""Normalize block_ids to list format."""
if not block_ids:
return []
if isinstance(block_ids, tuple):
return block_ids[0] if block_ids else []
if isinstance(block_ids, list):
return block_ids
return []
@dataclass
class HF3FSRequestMetadata:
"""Metadata for a single request in HF3FS connector."""
request_id: str
token_ids: list[int]
block_ids: list[int]
load_block_op: LoadBlockInfo | None = None
save_block_op: SaveBlockInfo | None = None
@staticmethod
def from_scheduling_state(
state: "RequestSchedulingState",
block_size: int,
load_op: LoadBlockInfo | None = None,
skip_leading_blocks: int | None = None,
) -> Optional["HF3FSRequestMetadata"]:
"""Create request metadata from scheduling state."""
token_count = len(state.token_ids)
total_blocks = token_count // block_size
skip_blocks = (
state.num_saved_blocks
if skip_leading_blocks is None
else skip_leading_blocks
)
new_blocks_to_save = total_blocks - state.num_saved_blocks
if new_blocks_to_save <= 0 and load_op is None:
return None
state.num_saved_blocks = total_blocks
return HF3FSRequestMetadata(
request_id=state.request_id,
token_ids=state.token_ids,
block_ids=state.allocated_block_ids,
load_block_op=load_op,
save_block_op=SaveBlockInfo(skip_leading_blocks=skip_blocks),
)
class HF3FSConnectorMetadata(KVConnectorMetadata):
"""Container for HF3FS connector metadata."""
def __init__(self):
self.requests: list[HF3FSRequestMetadata] = []
def add_request(self, request_metadata: HF3FSRequestMetadata) -> None:
"""Add request to metadata."""
self.requests.append(request_metadata)
@@ -1,262 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
@triton.jit
def kv_cache_scatter_kernel(
kv_cache_ptrs_ptr,
source_ptr,
token_indices_ptr,
num_tokens_in_block,
total_token_in_kvcache,
num_layers,
tokens_per_block,
block_stride,
head_stride,
token_stride,
content_stride,
num_heads: tl.constexpr,
content_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
layer_idx = tl.program_id(0)
token_pos = tl.program_id(1)
if layer_idx >= num_layers or token_pos >= num_tokens_in_block:
return
token_idx = tl.load(token_indices_ptr + token_pos)
kv_cache_ptr = tl.cast(tl.load(kv_cache_ptrs_ptr + layer_idx), source_ptr.dtype)
if token_idx >= total_token_in_kvcache:
return
block_idx = token_idx // tokens_per_block
state_idx = token_idx % tokens_per_block
hidden_size = num_heads * content_size
for i in range(0, hidden_size, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < hidden_size
head_idx = offset // content_size
content_idx = offset % content_size
source_offset = (
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
) * content_size + content_idx
target_offset = (
block_idx * block_stride
+ head_idx * head_stride
+ state_idx * token_stride
+ content_idx * content_stride
)
value = tl.load(source_ptr + source_offset, mask=mask)
tl.store(kv_cache_ptr + target_offset, value, mask=mask)
@triton.jit
def kv_cache_gather_kernel(
kv_cache_ptrs_ptr,
dst_ptr,
token_indices_ptr,
num_tokens_in_block,
total_token_in_kvcache,
num_layers,
tokens_per_block,
block_stride,
head_stride,
token_stride,
content_stride,
num_heads: tl.constexpr,
content_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
layer_idx = tl.program_id(0)
token_pos = tl.program_id(1)
if layer_idx >= num_layers or token_pos >= num_tokens_in_block:
return
token_idx = tl.load(token_indices_ptr + token_pos)
kv_cache_ptr = tl.cast(tl.load(kv_cache_ptrs_ptr + layer_idx), dst_ptr.dtype)
if token_idx >= total_token_in_kvcache:
return
block_idx = token_idx // tokens_per_block
state_idx = token_idx % tokens_per_block
hidden_size = num_heads * content_size
for i in range(0, hidden_size, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < hidden_size
head_idx = offset // content_size
content_idx = offset % content_size
kvcache_offset = (
block_idx * block_stride
+ head_idx * head_stride
+ state_idx * token_stride
+ content_idx * content_stride
)
dst_offset = (
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
) * content_size + content_idx
value = tl.load(kv_cache_ptr + kvcache_offset, mask=mask)
tl.store(dst_ptr + dst_offset, value, mask=mask)
def scatter_kv_caches(
kv_caches_ptrs: torch.Tensor,
total_token_in_kvcache: int,
src_tensor: torch.Tensor,
token_indices: list[int],
tokens_per_block: int,
num_heads: int,
content_size: int,
kv_cache_strides: tuple[int, ...],
) -> None:
"""Scatter KV cache data from source tensor to KV cache storage.
Args:
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
total_token_in_kvcache: Total number of tokens in KV cache
src_tensor: Source ``[L, H, N, C]`` tensor containing data to scatter
token_indices: List of token positions to update
tokens_per_block: Number of stored states in each cache block
num_heads: Size of the H axis
content_size: Size of the C axis
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
"""
num_layers = len(kv_caches_ptrs)
num_tokens_in_block = len(token_indices)
assert src_tensor.shape == (
num_layers,
num_heads,
num_tokens_in_block,
content_size,
)
assert len(kv_cache_strides) == 4
device = src_tensor.device
token_indices_tensor = async_tensor_h2d(
token_indices, device=device, dtype=torch.int32
)
grid = (num_layers, num_tokens_in_block)
BLOCK_SIZE = 128
kv_cache_scatter_kernel[grid](
kv_caches_ptrs,
src_tensor,
token_indices_tensor,
num_tokens_in_block,
total_token_in_kvcache,
num_layers,
tokens_per_block,
*kv_cache_strides,
num_heads=num_heads,
content_size=content_size,
BLOCK_SIZE=BLOCK_SIZE,
)
def gather_kv_caches(
kv_caches_ptrs: torch.Tensor,
total_token_in_kvcache: int,
dst_tensor: torch.Tensor,
token_indices: list[int],
tokens_per_block: int,
num_heads: int,
content_size: int,
kv_cache_strides: tuple[int, ...],
) -> None:
"""Gather KV cache data from KV cache storage to destination tensor.
Args:
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
total_token_in_kvcache: Total number of tokens in KV cache
dst_tensor: Destination ``[L, H, N, C]`` tensor
token_indices: List of token positions to gather
tokens_per_block: Number of stored states in each cache block
num_heads: Size of the H axis
content_size: Size of the C axis
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
"""
num_layers = kv_caches_ptrs.shape[0]
num_tokens_in_block = len(token_indices)
assert dst_tensor.shape == (
num_layers,
num_heads,
num_tokens_in_block,
content_size,
)
assert len(kv_cache_strides) == 4
device = dst_tensor.device
token_indices_tensor = async_tensor_h2d(
token_indices, device=device, dtype=torch.int32
)
grid = (num_layers, num_tokens_in_block)
BLOCK_SIZE = 128
kv_cache_gather_kernel[grid](
kv_caches_ptrs,
dst_tensor,
token_indices_tensor,
num_tokens_in_block,
total_token_in_kvcache,
num_layers,
tokens_per_block,
*kv_cache_strides,
num_heads=num_heads,
content_size=content_size,
BLOCK_SIZE=BLOCK_SIZE,
)
class CopyBufferAllocator:
"""Memory pool for tensor buffers to avoid frequent allocation/deallocation."""
def __init__(
self, device: torch.device, dtype: torch.dtype, shape: list, max_count: int
):
self._shape = shape
self._max_count = max_count
self._device = device
self._free_buffers = [
torch.empty(shape, dtype=dtype, device=device) for _ in range(max_count)
]
self._inuse_count = 0
def alloc_buffer(self, count: int) -> list[torch.Tensor] | None:
"""Allocate buffers from the pool."""
if count == 0:
return []
if self._inuse_count + count <= self._max_count:
self._inuse_count += count
result = self._free_buffers[-count:]
del self._free_buffers[-count:]
return result
return None
def free_buffer(self, buffers: list[torch.Tensor]) -> None:
"""Return buffers to the pool."""
if not buffers:
return
if self._inuse_count >= len(buffers):
self._inuse_count -= len(buffers)
self._free_buffers.extend(buffers)
else:
raise RuntimeError("Attempted to free more buffers than allocated")
logger = init_logger(__name__)
@@ -1,133 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import os
import torch
logger = logging.getLogger(__name__)
HF3FS_AVAILABLE = True
class Hf3fsClient:
"""Mock HF3FS client using file backend for debugging and testing."""
def __init__(self, path: str, size: int, bytes_per_page: int, entries: int):
self._size = size
self._bytes_per_page = bytes_per_page
self._entries = entries
self._file_path = path
self._ensure_file_exists()
logger.debug("Initialized mock HF3FS client: %s (%d bytes)", path, size)
def _ensure_file_exists(self) -> None:
"""Create file if it doesn't exist."""
if not os.path.exists(self._file_path):
with open(self._file_path, "w+b") as f:
f.truncate(self._size)
def batch_read(self, offsets: list[int], tensors: list[torch.Tensor]) -> list[int]:
"""Read data from file at specified offsets into tensors."""
results = []
try:
with open(self._file_path, "rb") as f:
for offset, tensor in zip(offsets, tensors):
num_bytes = tensor.numel() * tensor.element_size()
if offset < 0 or offset + num_bytes > self._size:
results.append(-1)
continue
f.seek(offset)
buffer_data = f.read(num_bytes)
if len(buffer_data) == num_bytes == self._bytes_per_page:
tensor_data = self._convert_buffer_to_tensor(
buffer_data, tensor.dtype
)
tensor.copy_(
tensor_data.reshape(tensor.shape).to(tensor.device)
)
results.append(self._bytes_per_page)
else:
logger.error(
"Read size mismatch: got %d, expected %d",
len(buffer_data),
num_bytes,
)
results.append(-1)
except Exception as e:
logger.error("Batch read error: %s", e)
results.extend([-1] * (len(offsets) - len(results)))
return results
def _convert_buffer_to_tensor(
self, buffer_data: bytes, dtype: torch.dtype
) -> torch.Tensor:
"""Convert buffer data to tensor with proper dtype handling."""
if dtype == torch.bfloat16:
tensor_data = torch.frombuffer(buffer_data, dtype=torch.uint16)
return tensor_data.view(dtype=torch.bfloat16)
else:
return torch.frombuffer(buffer_data, dtype=dtype)
def batch_write(
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
) -> list[int]:
"""Write data from tensors to file at specified offsets."""
results = []
try:
torch.cuda.current_stream().wait_event(event)
# Convert tensors to bytes
data_bytes_list = [self._tensor_to_bytes(tensor) for tensor in tensors]
# Write to file
with open(self._file_path, "r+b") as f:
for offset, data_bytes in zip(offsets, data_bytes_list):
if offset < 0 or offset + len(data_bytes) > self._size:
results.append(-1)
continue
f.seek(offset)
bytes_written = f.write(data_bytes)
if bytes_written == len(data_bytes) == self._bytes_per_page:
results.append(self._bytes_per_page)
else:
logger.error(
"Write size mismatch: wrote %d, expected %d",
bytes_written,
self._bytes_per_page,
)
results.append(-1)
except Exception as e:
logger.error("Batch write error: %s", e)
results.extend([-1] * (len(offsets) - len(results)))
return results
def _tensor_to_bytes(self, tensor: torch.Tensor) -> bytes:
"""Convert tensor to bytes with proper dtype handling."""
cpu_tensor = tensor.cpu()
if cpu_tensor.dtype == torch.bfloat16:
return cpu_tensor.view(dtype=torch.uint16).numpy().tobytes()
else:
return cpu_tensor.numpy().tobytes()
def get_size(self) -> int:
"""Get the total size of the storage file."""
return self._size
def close(self) -> None:
"""Close the client (no-op for file backend)."""
pass
def flush(self) -> None:
"""Flush any pending writes (no-op for file backend)."""
pass
@@ -1,57 +0,0 @@
#include <cuda_runtime.h>
#include <torch/extension.h>
#include <cstring>
#include <vector>
void read_shm(const torch::Tensor& shm, const torch::Tensor& pin,
std::vector<torch::Tensor> dst, uint64_t stream_ptr) {
py::gil_scoped_release release;
cudaStream_t stream = reinterpret_cast<cudaStream_t>(stream_ptr);
// Copy from shared memory to pinned memory
char* shm_ptr = static_cast<char*>(shm.data_ptr());
char* src_ptr = static_cast<char*>(pin.data_ptr());
std::memcpy(src_ptr, shm_ptr, shm.numel() * shm.element_size());
// Copy from pinned memory to GPU tensors
size_t current = 0;
for (size_t i = 0; i < dst.size(); ++i) {
auto& t = dst[i];
size_t t_bytes = t.numel() * t.element_size();
char* dst_ptr = static_cast<char*>(t.data_ptr());
cudaMemcpyAsync(dst_ptr, src_ptr + current, t_bytes, cudaMemcpyHostToDevice,
stream);
current += t_bytes;
}
cudaStreamSynchronize(stream);
}
void write_shm(const std::vector<torch::Tensor> src, torch::Tensor& shm,
const torch::Tensor& pin, uint64_t stream_ptr) {
py::gil_scoped_release release;
cudaStream_t stream = reinterpret_cast<cudaStream_t>(stream_ptr);
// Copy from GPU tensors to pinned memory
char* dst_ptr = static_cast<char*>(pin.data_ptr());
size_t current = 0;
for (size_t i = 0; i < src.size(); ++i) {
auto& t = src[i];
size_t t_bytes = t.numel() * t.element_size();
char* src_ptr = static_cast<char*>(t.data_ptr());
cudaMemcpyAsync(dst_ptr + current, src_ptr, t_bytes, cudaMemcpyDeviceToHost,
stream);
current += t_bytes;
}
cudaStreamSynchronize(stream);
// Copy from pinned memory to shared memory
char* shm_ptr = static_cast<char*>(shm.data_ptr());
std::memcpy(shm_ptr, dst_ptr, shm.numel() * shm.element_size());
}
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
m.def("read_shm", &read_shm, "Read tensors from shared memory");
m.def("write_shm", &write_shm, "Write tensors to shared memory");
}
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,408 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Connector boundary for HiSparse scheduler and worker state."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorWorkerMetadata,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparseKVConnectorStats,
HiSparsePromMetrics,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.layout import get_hisparse_steady_state_concurrency
from vllm.v1.hisparse.types import (
SparseKVOffloadCommand,
SparseKVResidencyUpdate,
SparseKVRowMirror,
)
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.forward_context import ForwardContext
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.hisparse.coordinator import HiSparseCoordinator
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
@dataclass
class HiSparseConnectorMetadata(KVConnectorMetadata):
command: SparseKVOffloadCommand | None
host_block_copies: tuple[KVCacheBlockCopy, ...]
source_block_ids: tuple[int, ...]
row_mirrors: dict[str, tuple[SparseKVRowMirror, ...]]
all_context_pages_resident: bool
residency_updates: dict[str, SparseKVResidencyUpdate]
@dataclass
class HiSparseConnectorWorkerMetadata(KVConnectorWorkerMetadata):
enqueued_transfer_counts: dict[int, int]
completed_transfer_counts: dict[int, int]
completed_host_copy_dst_ids: tuple[int, ...] = ()
def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetadata:
assert isinstance(other, HiSparseConnectorWorkerMetadata)
def add_counts(first: dict[int, int], second: dict[int, int]) -> dict[int, int]:
combined = first.copy()
for transfer_id, count in second.items():
combined[transfer_id] = combined.get(transfer_id, 0) + count
return combined
return HiSparseConnectorWorkerMetadata(
enqueued_transfer_counts=add_counts(
self.enqueued_transfer_counts, other.enqueued_transfer_counts
),
completed_transfer_counts=add_counts(
self.completed_transfer_counts, other.completed_transfer_counts
),
# Every worker runs the same copies, so a union is the ack.
completed_host_copy_dst_ids=tuple(
{*self.completed_host_copy_dst_ids, *other.completed_host_copy_dst_ids}
),
)
class HiSparseConnectorScheduler:
def __init__(
self,
*,
async_speculative: bool,
draft_kv_lookahead: int = 0,
) -> None:
self.coordinator: HiSparseCoordinator | None = None
self.async_speculative = async_speculative
self.draft_kv_lookahead = draft_kv_lookahead
self.requests: dict[str, Request] = {}
def bind_coordinator(self, coordinator: HiSparseCoordinator) -> None:
assert self.coordinator is None
self.coordinator = coordinator
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
if self.coordinator is None:
return None
host_pool = self.coordinator.get_host_block_pool()
assert host_pool is not None
stats = HiSparseKVConnectorStats()
stats.record_host_usage(
host_pool.get_usage(), len(self.coordinator.pending_spills)
)
if host_pool.metrics_collector is not None:
stats.record_host_evictions(host_pool.metrics_collector.drain_events())
return stats
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> HiSparseConnectorMetadata:
assert self.coordinator is not None
# HiSparse rebinds worker state every step, so the load must start
# before the forward rather than being deferred to post-forward.
scheduler_output.has_sync_kv_loads = True
num_computed_tokens = {
request.req_id: request.num_computed_tokens
for request in scheduler_output.scheduled_new_reqs
}
num_computed_tokens.update(
zip(
scheduler_output.scheduled_cached_reqs.req_ids,
scheduler_output.scheduled_cached_reqs.num_computed_tokens,
)
)
scheduled_requests = tuple(
(
request_id,
num_computed_tokens[request_id],
scheduled_count,
)
for request_id, scheduled_count in (
scheduler_output.num_scheduled_tokens.items()
)
)
# Runs first: it plans this step's transfers and residency changes.
self.coordinator.advance_scheduled(
(
request_id,
min(start + count, self.requests[request_id].num_tokens),
)
for request_id, start, count in scheduled_requests
)
command = self.coordinator.build_offload_command()
host_block_copies = self.coordinator.take_host_block_copies()
source_group_id = self.coordinator.host_group_id
assert source_group_id is not None
source_block_ids = [
block_id
for request in scheduler_output.scheduled_new_reqs
for block_id in request.block_ids[source_group_id]
]
for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids:
if new_block_ids is not None:
source_block_ids.extend(new_block_ids[source_group_id])
row_mirrors = {}
for request_id, scheduled_start, scheduled_count in scheduled_requests:
mirror_start = scheduled_start
if self.async_speculative:
mirror_start = max(
0,
scheduled_start - self.requests[request_id].num_output_placeholders,
)
row_mirrors[request_id] = self.coordinator.build_row_mirrors(
(
(
request_id,
mirror_start,
scheduled_count
+ scheduled_start
- mirror_start
+ self.draft_kv_lookahead,
),
),
)
return HiSparseConnectorMetadata(
command,
host_block_copies,
tuple(source_block_ids),
row_mirrors,
self.coordinator.all_context_pages_resident(scheduled_requests),
self.coordinator.take_residency_updates(
scheduler_output.num_scheduled_tokens
),
)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
assert self.coordinator is not None
for request_id in connector_output.finished_recving or ():
self.coordinator.finish_host_import(
request_id, failed=request_id in connector_output.failed_recving
)
metadata = connector_output.kv_connector_worker_meta
if metadata is None:
return
assert isinstance(metadata, HiSparseConnectorWorkerMetadata)
self.coordinator.release_completed_host_copies(
metadata.completed_host_copy_dst_ids
)
self.coordinator.update_spills(
metadata.enqueued_transfer_counts,
metadata.completed_transfer_counts,
)
class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
"""Join the scheduler coordinator to the worker's transfer engine."""
@classmethod
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str:
return "BLHNC"
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig,
) -> None:
super().__init__(vllm_config, role, kv_cache_config)
if kv_cache_config.hisparse_host_num_blocks is None:
raise ValueError("HiSparseConnector requires a HiSparse host pool")
self.connector_scheduler: HiSparseConnectorScheduler | None = None
self.connector_worker: HiSparseConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
speculative_config = vllm_config.speculative_config
self.connector_scheduler = HiSparseConnectorScheduler(
async_speculative=bool(
vllm_config.scheduler_config.async_scheduling
and speculative_config is not None
),
draft_kv_lookahead=vllm_config.num_lookahead_tokens,
)
max_model_len = vllm_config.model_config.max_model_len
steady_concurrency = get_hisparse_steady_state_concurrency(
vllm_config, kv_cache_config
)
logger.info_once(
"HiSparse steady-state maximum concurrency for %s tokens per "
"request: %.2fx (running requests reading from host).",
f"{max_model_len:,}",
steady_concurrency,
)
elif role == KVConnectorRole.WORKER:
self.connector_worker = HiSparseConnectorWorker(
vllm_config, kv_cache_config
)
else:
raise ValueError(f"Unsupported KV connector role: {role}")
def bind_kv_cache_manager(self, kv_cache_manager: KVCacheManager) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_coordinator(
get_hisparse_coordinator(kv_cache_manager)
)
@property
def requires_kv_delivery(self) -> bool:
return False
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
assert self.connector_scheduler.coordinator is not None
return self.connector_scheduler.coordinator.has_pending_work()
def has_pending_block_frees(self) -> bool:
assert self.connector_scheduler is not None
assert self.connector_scheduler.coordinator is not None
return self.connector_scheduler.coordinator.has_pending_reclamation()
def finish_forward(self) -> None:
assert self.connector_worker is not None
self.connector_worker.finish_forward()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def reset_capture_state(self) -> None:
assert self.connector_worker is not None
self.connector_worker.reset_hot_state()
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is not None:
return self.connector_worker.get_kv_connector_stats()
if self.connector_scheduler is not None:
return self.connector_scheduler.get_kv_connector_stats()
return None
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
HiSparseKVConnectorStats(data=data)
if data is not None
else HiSparseKVConnectorStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return HiSparsePromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, HiSparseConnectorMetadata)
request_state_indices = kwargs.get("request_state_indices")
assert request_state_indices is None or isinstance(
request_state_indices, torch.Tensor
)
request_ids = kwargs.get("request_ids")
assert request_ids is None or isinstance(request_ids, list)
attn_metadata = kwargs.get("attn_metadata")
assert attn_metadata is None or isinstance(attn_metadata, dict)
self.connector_worker.start_step(
metadata,
request_state_indices,
request_ids,
num_tokens=int(kwargs.get("num_tokens") or 0),
)
self.connector_worker.prepare_forward(attn_metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
return
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
return
def wait_for_save(self) -> None:
return
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
assert self.connector_worker is not None
enqueued, completed = self.connector_worker.take_transfer_updates()
host_copies = self.connector_worker.take_completed_host_copies()
if not enqueued and not completed and not host_copies:
return None
return HiSparseConnectorWorkerMetadata(
enqueued_transfer_counts={transfer_id: 1 for transfer_id in enqueued},
completed_transfer_counts={transfer_id: 1 for transfer_id in completed},
completed_host_copy_dst_ids=tuple(host_copies),
)
def shutdown(self) -> None:
if self.connector_worker is not None:
self.connector_worker.shutdown()
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
return 0, False
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.requests[request.request_id] = request
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
def request_finished_all_groups(
self,
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
self.connector_scheduler.requests.pop(request.request_id, None)
return False, None
@@ -1,214 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stats and Prometheus metrics for the HiSparse connector."""
from dataclasses import dataclass
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.v1.metrics.buckets import histogram_buckets
from vllm.v1.metrics.stats import KVCacheEvictionEvent
from vllm.v1.metrics.utils import create_metric_per_engine
_HISPARSE_LEVEL_KEYS = frozenset({"host_cache_usage_perc", "pending_page_transfers"})
@dataclass
class HiSparseKVConnectorStats(KVConnectorStats):
"""Container for HiSparse hot-buffer residency metrics.
Each list entry is the delta recorded over one device counter snapshot
interval, so a list can hold multiple snapshots per logging interval.
The host-tier gauges are level values, not deltas: aggregation keeps
only the most recent observation.
"""
def __post_init__(self):
if not self.data:
# Empty container init, no data is passed in.
self.reset()
def reset(self):
# Must be serializable
self.data: dict[str, list[int | float]] = {
"cache_hits": [],
"cache_misses": [],
"host_to_device_bytes": [],
"host_cache_usage_perc": [],
"pending_page_transfers": [],
"host_block_lifetime_seconds": [],
"host_block_idle_before_evict_seconds": [],
"host_block_reuse_gap_seconds": [],
}
def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int):
self.data["cache_hits"].append(hits)
self.data["cache_misses"].append(misses)
self.data["host_to_device_bytes"].append(host_to_device_bytes)
def record_host_usage(self, usage: float, pending_page_transfers: int) -> None:
self.data["host_cache_usage_perc"].append(usage)
self.data["pending_page_transfers"].append(pending_page_transfers)
def record_host_evictions(self, events: list[KVCacheEvictionEvent]) -> None:
for event in events:
self.data["host_block_lifetime_seconds"].append(event.lifetime_seconds)
self.data["host_block_idle_before_evict_seconds"].append(event.idle_seconds)
self.data["host_block_reuse_gap_seconds"].extend(event.reuse_gaps_seconds)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
accumulator = self.data[k]
assert isinstance(accumulator, list)
if k in _HISPARSE_LEVEL_KEYS:
accumulator.extend(v[-1:])
else:
accumulator.extend(v)
return self
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging.
reduced: dict[str, int | float] = {
"HiSparse hot-buffer hits": sum(self.data["cache_hits"]),
"HiSparse hot-buffer misses": sum(self.data["cache_misses"]),
"HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]),
}
if self.data["host_cache_usage_perc"]:
usage = self.data["host_cache_usage_perc"][-1]
reduced["HiSparse host KV cache usage %"] = round(100 * usage, 1)
return reduced
def is_empty(self) -> bool:
return all(len(values) == 0 for values in self.data.values())
class HiSparsePromMetrics(KVConnectorPromMetrics):
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
counter_cache_hits = self._counter_cls(
name="vllm:hisparse_cache_hits",
documentation="Number of HiSparse device hot-buffer hits.",
labelnames=labelnames,
)
counter_cache_misses = self._counter_cls(
name="vllm:hisparse_cache_misses",
documentation="Number of HiSparse device hot-buffer misses.",
labelnames=labelnames,
)
counter_host_to_device_bytes = self._counter_cls(
name="vllm:hisparse_host_to_device_bytes",
documentation=(
"Bytes transferred from host KV storage to HiSparse hot buffers."
),
labelnames=labelnames,
)
self.hisparse_counters: dict[str, Any] = {
"cache_hits": create_metric_per_engine(
counter_cache_hits, self.per_engine_labelvalues
),
"cache_misses": create_metric_per_engine(
counter_cache_misses, self.per_engine_labelvalues
),
"host_to_device_bytes": create_metric_per_engine(
counter_host_to_device_bytes, self.per_engine_labelvalues
),
}
gauge_host_cache_usage = self._gauge_cls(
name="vllm:hisparse_host_cache_usage_perc",
documentation=(
"HiSparse host KV-cache usage. 1 means 100 percent usage. "
"Evictable cached blocks count as free."
),
multiprocess_mode="mostrecent",
labelnames=labelnames,
)
gauge_pending_page_transfers = self._gauge_cls(
name="vllm:hisparse_pending_page_transfers",
documentation=(
"HiSparse page transfers (write-backs and restores) not yet completed."
),
multiprocess_mode="mostrecent",
labelnames=labelnames,
)
self.hisparse_gauges: dict[str, Any] = {
"host_cache_usage_perc": create_metric_per_engine(
gauge_host_cache_usage, self.per_engine_labelvalues
),
"pending_page_transfers": create_metric_per_engine(
gauge_pending_page_transfers, self.per_engine_labelvalues
),
}
self.hisparse_histograms: dict[str, Any] = {}
if vllm_config.observability_config.kv_cache_metrics:
residency_buckets = histogram_buckets(
"kv_cache_residency",
overrides=vllm_config.observability_config.custom_histogram_buckets,
)
histogram_host_block_lifetime = self._histogram_cls(
name="vllm:hisparse_host_block_lifetime_seconds",
documentation=(
"Histogram of HiSparse host KV block lifetime from allocation "
"to eviction. Sampled metrics (controlled by "
"--kv-cache-metrics-sample)."
),
buckets=residency_buckets,
labelnames=labelnames,
)
histogram_host_block_idle_before_evict = self._histogram_cls(
name="vllm:hisparse_host_block_idle_before_evict_seconds",
documentation=(
"Histogram of HiSparse host KV block idle time before eviction. "
"Sampled metrics (controlled by --kv-cache-metrics-sample)."
),
buckets=residency_buckets,
labelnames=labelnames,
)
histogram_host_block_reuse_gap = self._histogram_cls(
name="vllm:hisparse_host_block_reuse_gap_seconds",
documentation=(
"Histogram of time gaps between consecutive HiSparse host KV "
"block accesses. Sampled metrics (controlled by "
"--kv-cache-metrics-sample)."
),
buckets=residency_buckets,
labelnames=labelnames,
)
self.hisparse_histograms = {
"host_block_lifetime_seconds": create_metric_per_engine(
histogram_host_block_lifetime, self.per_engine_labelvalues
),
"host_block_idle_before_evict_seconds": create_metric_per_engine(
histogram_host_block_idle_before_evict, self.per_engine_labelvalues
),
"host_block_reuse_gap_seconds": create_metric_per_engine(
histogram_host_block_reuse_gap, self.per_engine_labelvalues
),
}
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
for name, counter in self.hisparse_counters.items():
for value in transfer_stats_data.get(name, []):
counter[engine_idx].inc(value)
for name, gauge in self.hisparse_gauges.items():
values = transfer_stats_data.get(name, [])
if values:
gauge[engine_idx].set(values[-1])
for name, histogram in self.hisparse_histograms.items():
for value in transfer_stats_data.get(name, []):
histogram[engine_idx].observe(value)
@@ -1,978 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HiSparse worker-side host/hot state and data movement."""
from __future__ import annotations
from collections import deque
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any, NamedTuple
import numpy as np
import torch
from vllm import _custom_ops as ops
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparseKVConnectorStats,
)
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tp_group,
)
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import current_stream
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
from vllm.v1.hisparse.runtime import (
HiSparseCacheHandle,
drop_hisparse_host_pool,
release_pinned_state,
update_hisparse_residency,
)
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
from vllm.v1.kv_cache_interface import (
HiSparseHotSpec,
KVCacheConfig,
)
from vllm.v1.worker.utils import copy_kv_cache_blocks_inplace
if TYPE_CHECKING:
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnectorMetadata,
)
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
class _DMADescriptors(NamedTuple):
src: torch.Tensor
dst: torch.Tensor
sizes: torch.Tensor
src_np: np.ndarray
dst_np: np.ndarray
sizes_np: np.ndarray
@dataclass
class _SlotMappingStaging:
stream: torch.Stream
event: torch.Event
slots: torch.Tensor
candidates: tuple[SparseKVRowMirror, ...] = ()
num_tokens: int = 0
source_index: int = 0
def _allocate_dma_descriptors(size: int) -> _DMADescriptors:
src, dst, sizes = (torch.empty(size, dtype=torch.int64) for _ in range(3))
return _DMADescriptors(
src,
dst,
sizes,
src.numpy(),
dst.numpy(),
sizes.numpy(),
)
_METRICS_INTERVAL = 2000
def _get_hisparse_cache(
forward_context: dict[str, Any], layer_name: str
) -> HiSparseCacheHandle:
attention_layer = forward_context[layer_name]
hisparse_cache = attention_layer.hisparse_cache
assert hisparse_cache is not None
return hisparse_cache
def _flatten_row_mirrors(
row_mirrors: Mapping[str, tuple[SparseKVRowMirror, ...]],
request_ids: Sequence[str] | None,
) -> tuple[SparseKVRowMirror, ...]:
ordered_ids = row_mirrors if request_ids is None else request_ids
return tuple(
mirror
for request_id in ordered_ids
for mirror in row_mirrors.get(request_id, ())
)
def _select_written_row_mirrors(
candidates: tuple[SparseKVRowMirror, ...],
source_slots: np.ndarray,
source_index: int,
) -> tuple[SparseKVRowMirror, ...]:
"""Select and coalesce GPU-written rows from a scheduler-owned envelope."""
source_slots = source_slots[source_slots >= 0]
if source_slots.size == 0 or not candidates:
return ()
starts = np.asarray([mirror.source_starts for mirror in candidates], dtype=np.int64)
counts = np.fromiter((mirror.num_rows for mirror in candidates), dtype=np.int64)
destinations = np.fromiter(
[mirror.destination_start for mirror in candidates], dtype=np.int64
)
if source_index >= starts.shape[1]:
raise RuntimeError("HiSparse row-mirror source index is out of range.")
source_starts = starts[:, source_index]
order = np.argsort(source_starts)
matches = np.searchsorted(source_starts[order], source_slots, side="right") - 1
candidate_ids = order[np.maximum(matches, 0)]
offsets = source_slots - source_starts[candidate_ids]
valid = (matches >= 0) & (offsets < counts[candidate_ids])
if not np.all(valid):
# A page the scheduler could not map has no host destination, so its
# rows are simply not mirrorable this step.
source_slots = source_slots[valid]
candidate_ids = candidate_ids[valid]
offsets = offsets[valid]
if source_slots.size == 0:
return ()
sources = starts[candidate_ids] + offsets[:, None]
destinations = destinations[candidate_ids] + offsets
boundaries = (
np.flatnonzero(
(candidate_ids[1:] != candidate_ids[:-1])
| (np.diff(destinations) != 1)
| np.any(np.diff(sources, axis=0) != 1, axis=1)
)
+ 1
)
run_starts = np.concatenate(([0], boundaries))
run_ends = np.concatenate((boundaries, [destinations.size]))
return tuple(
SparseKVRowMirror(
source_starts=tuple(int(value) for value in sources[start]),
destination_start=int(destinations[start]),
num_rows=int(end - start),
)
for start, end in zip(run_starts, run_ends, strict=True)
)
def _is_hisparse_host_writer(
shared_host_region: SharedOffloadRegion | None,
) -> bool:
return shared_host_region is None or get_tensor_model_parallel_rank() == 0
def _create_hisparse_host_events(
shared_host_region: SharedOffloadRegion | None,
is_host_writer: bool,
device: torch.device,
) -> tuple[torch.Event, torch.Event]:
if shared_host_region is None:
return torch.Event(), torch.Event()
events: tuple[torch.Event, torch.Event] | None = None
ipc_handles = None
if is_host_writer:
events = (
torch.cuda.Event(interprocess=True),
torch.cuda.Event(interprocess=True),
)
stream = current_stream()
for event in events:
event.record(stream)
ipc_handles = tuple(event.ipc_handle() for event in events)
ipc_handles = get_tp_group().broadcast_object(ipc_handles, src=0)
if events is None:
events = (
torch.cuda.Event.from_ipc_handle(device, ipc_handles[0]),
torch.cuda.Event.from_ipc_handle(device, ipc_handles[1]),
)
return events
class HiSparseConnectorWorker:
"""Own HiSparse host/hot state and execute its worker-side transfers."""
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
self._initialized = False
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
forward_context = self.vllm_config.compilation_config.static_forward_context
cache_handles: list[HiSparseCacheHandle] = []
cache_layer_names: list[str] = []
for group in self.kv_cache_config.kv_cache_groups:
if not isinstance(group.kv_cache_spec, HiSparseHotSpec):
continue
for cache_name in group.layer_names:
assert cache_name.endswith(HISPARSE_HOT_SUFFIX)
layer_name = cache_name[: -len(HISPARSE_HOT_SUFFIX)]
cache_handles.append(_get_hisparse_cache(forward_context, layer_name))
cache_layer_names.append(layer_name)
if not cache_handles:
raise RuntimeError("HiSparse connector found no hot-cache handles.")
hot_backings: dict[int, torch.Tensor] = {}
registered_host_pools: dict[int, torch.Tensor] = {}
shared_host_regions: dict[int, SharedOffloadRegion] = {}
for cache in cache_handles:
hot_backing = cache.runtime.hot_backing
hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing
registered_pool = cache.runtime.registered_host_pool
registered_host_pools[registered_pool.data_ptr()] = registered_pool
if (region := cache.runtime.shared_host_region) is not None:
shared_host_regions[id(region)] = region
if len(hot_backings) != 1:
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
hot_backing = next(iter(hot_backings.values()))
if len(shared_host_regions) > 1:
raise RuntimeError("HiSparse caches must share one host region.")
shared_host_region = next(iter(shared_host_regions.values()), None)
pinned_host_pools = (
[]
if shared_host_region is not None
else list(registered_host_pools.values())
)
is_host_writer = _is_hisparse_host_writer(shared_host_region)
resident = cache_handles[0].view
assert resident is not None
host_num_blocks = self.kv_cache_config.hisparse_host_num_blocks
assert host_num_blocks is not None
try:
self.initialize(
cache_handles,
cache_layer_names,
hot_backing,
self.vllm_config.scheduler_config.max_num_seqs,
host_num_blocks,
hot_backing.device,
pinned_host_pools,
shared_host_region=shared_host_region,
is_host_writer=is_host_writer,
)
except Exception:
release_pinned_state(
[cache.runtime for cache in cache_handles],
pinned_host_pools,
shared_host_region,
)
drop_hisparse_host_pool(self.vllm_config)
raise
def initialize(
self,
cache_handles: list[HiSparseCacheHandle],
cache_layer_names: list[str],
hot_backing: torch.Tensor,
max_num_reqs: int,
host_num_blocks: int,
device: torch.device,
pinned_host_pools: list[torch.Tensor],
*,
shared_host_region: SharedOffloadRegion | None = None,
is_host_writer: bool = True,
) -> None:
if self._initialized:
raise RuntimeError("HiSparse connector worker is already initialized.")
resident = cache_handles[0].view
assert resident is not None
self.kernel_block_size = resident.block_size
self.host_num_blocks = host_num_blocks
self.pinned_host_pools = pinned_host_pools
self.shared_host_region = shared_host_region
self.is_host_writer = is_host_writer
self.dma_stream = (
torch.cuda.Stream(device=device) if self.is_host_writer else None
)
self._slot_mapping_staging = None
if self.is_host_writer:
max_mirror_rows = (
self.vllm_config.scheduler_config.max_num_batched_tokens
+ max_num_reqs * (self.vllm_config.num_lookahead_tokens + 1)
)
self._slot_mapping_staging = _SlotMappingStaging(
stream=torch.Stream(device=device),
event=torch.Event(),
slots=torch.empty(
max_mirror_rows,
dtype=torch.int64,
pin_memory=True,
),
)
self._dma_free_descriptors: list[_DMADescriptors] = []
self._pending_dma_descriptors: deque[tuple[torch.Event, _DMADescriptors]] = (
deque()
)
self._dma_submitted = False
self._per_layer_mirrored: set[int] = set()
self._submitted_mirror_layers: set[int] = set()
self._layer_ready_events = tuple(torch.Event() for _ in cache_handles)
self._forward_ready_event = torch.Event()
self._draft_layers = tuple(
index for index, cache in enumerate(cache_handles) if cache.draft_layer
)
self._draft_mirror_pending = False
self._set_row_mirrors(())
self.cache_layer_names = cache_layer_names
self._group_leaders = tuple(
(layer_name, cache)
for layer_name, cache in zip(cache_layer_names, cache_handles, strict=True)
if cache.runtime.is_group_leader
)
self.host_write_events = _create_hisparse_host_events(
shared_host_region, is_host_writer, device
)
self.host_write_event = self.host_write_events[1]
self._next_host_write_event = 0
self.cache_handles = cache_handles
self.leader_runtimes = [cache.runtime for _, cache in self._group_leaders]
request_state_indices = {
indices.data_ptr(): indices
for cache in cache_handles
if (indices := cache.runtime.request_state_indices) is not None
}
if len(request_state_indices) != 1:
raise RuntimeError(
"HiSparse runtimes must share one request-state mapping."
)
self.request_state_indices = next(iter(request_state_indices.values()))
if self.request_state_indices.numel() != max_num_reqs:
raise RuntimeError(
"HiSparse request-state mapping does not match max_num_seqs."
)
self.hot_backing = hot_backing
self.residency: torch.Tensor | None = next(
(cache.residency for cache in cache_handles if cache.residency is not None),
None,
)
self._pending_invalid_block_ids: list[int] = []
# Destination block ids of host copies this worker has run.
self._completed_host_copy_dst_ids: list[int] = []
self._post_forward_transfers: list[SparseKVPageTransfer] = []
self._enqueued_transfer_ids: list[int] = []
self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = (
deque()
)
self._metrics_calls = 0
self._metrics_event = torch.Event()
self._metrics_pending = False
self._init_dma()
if self.is_host_writer:
for layer_index, handle in enumerate(cache_handles):
# Draft layers are mirrored once in the next start_step instead.
if not handle.draft_layer:
handle.submit_layer_mirror = partial(
self._enqueue_layer_mirror, layer_index
)
self._initialized = True
def set_request_state_indices(self, indices: torch.Tensor) -> None:
if indices.numel() > self.request_state_indices.numel():
raise ValueError(
"HiSparse request-state mapping exceeds max_num_seqs: "
f"{indices.numel()} > {self.request_state_indices.numel()}."
)
if torch.cuda.is_current_stream_capturing():
return
# Attention indexes persistent request state by input-batch row. Refresh
# that indirection after every batch compaction or reorder.
self.request_state_indices.fill_(-1)
self.request_state_indices[: indices.numel()].copy_(indices)
if self._pending_invalid_block_ids:
self.invalidate_blocks(self._pending_invalid_block_ids, indices)
self._pending_invalid_block_ids.clear()
def _init_dma(self) -> None:
host_caches = tuple(cache.runtime.host_cache for cache in self.cache_handles)
resident_caches = []
for cache in self.cache_handles:
assert cache.view is not None and cache.slot_mapping is not None
resident_caches.append(cache.view.cache)
self.host_caches = host_caches
self.resident_caches = tuple(resident_caches)
for resident_cache, host_cache in zip(resident_caches, host_caches):
row_bytes = resident_cache.shape[-1] * resident_cache.element_size()
if (
resident_cache.ndim != 3
or resident_cache.shape[1] != self.kernel_block_size
or resident_cache.stride(1) * resident_cache.element_size() != row_bytes
):
raise RuntimeError("HiSparse DMA requires contiguous resident rows.")
if (
host_cache.ndim != 2
or not host_cache.is_contiguous()
or host_cache.shape[1] * host_cache.element_size() != row_bytes
):
raise RuntimeError("HiSparse DMA requires contiguous host rows.")
def start_step(
self,
metadata: HiSparseConnectorMetadata,
request_state_indices: torch.Tensor | None,
request_ids: list[str] | None = None,
num_tokens: int = 0,
) -> None:
self._stage_row_mirror_mapping(num_tokens)
self._finish_previous_step()
previous_host_write_event = self.host_write_event
self.host_write_event = self.host_write_events[self._next_host_write_event]
self._next_host_write_event ^= 1
current_stream().wait_event(previous_host_write_event)
if self.shared_host_region is not None and not num_tokens:
# Without a forward, no TP collective keeps rank 0 from recording
# a shared event again before every rank has issued its wait.
get_tp_group().barrier()
self._release_completed_dma_descriptors()
self._dma_submitted = False
mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids)
if self._slot_mapping_staging is not None:
self._slot_mapping_staging.candidates = mirrors
self._set_row_mirrors(mirrors)
self._clear_forward_mirror_state()
for handle in self.cache_handles:
handle.all_context_pages_resident = metadata.all_context_pages_resident
self._copy_host_blocks(metadata.host_block_copies, previous_host_write_event)
transfers = (
metadata.command.page_transfers if metadata.command is not None else []
)
self._restore_pages([transfer for transfer in transfers if transfer.restore])
self._post_forward_transfers = [
transfer
for transfer in transfers
if not transfer.restore and transfer.after_forward
]
self._submit_transfers(
[
transfer
for transfer in transfers
if not transfer.restore and not transfer.after_forward
]
)
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
if request_state_indices is not None:
self.set_request_state_indices(request_state_indices)
if metadata.residency_updates:
assert self.residency is not None
assert request_state_indices is not None and request_ids is not None
update_hisparse_residency(
self.residency,
metadata.residency_updates,
request_ids,
request_state_indices,
)
def _clear_forward_mirror_state(self) -> None:
self._per_layer_mirrored.clear()
self._submitted_mirror_layers.clear()
for handle in self.cache_handles:
handle.decode_batch = False
handle.host_mirror_required = False
handle.num_actual_tokens = 0
handle.num_decode_tokens = 0
handle.req_id_per_token = None
def prepare_forward(self, attn_metadata: Mapping[str, Any] | None) -> None:
if attn_metadata is None:
return
for layer_name, handle in self._group_leaders:
metadata = attn_metadata.get(layer_name)
if metadata is not None:
handle.prepare_group_for_batch(metadata)
def _stage_row_mirror_mapping(self, num_tokens: int) -> None:
"""Snapshot the rows this forward will write, off the compute stream.
The resident slot mapping is a persistent view bound at registration
time, so the rows are already staged by the time the forward launches.
"""
state = self._slot_mapping_staging
if state is None or not num_tokens:
return
handle = self.cache_handles[0]
slots = handle.slot_mapping
assert slots is not None
source_index = handle.runtime.resident_source_index
start = state.num_tokens
end = start + num_tokens
if end > state.slots.shape[0]:
raise ValueError(
"HiSparse row mapping exceeds staging capacity: "
f"{end} > {state.slots.shape[0]}."
)
main_stream = current_stream()
state.stream.wait_stream(main_stream)
with torch.cuda.stream(state.stream):
state.slots[start:end].copy_(
slots[:num_tokens],
non_blocking=True,
)
state.event.record(state.stream)
state.num_tokens = end
state.source_index = source_index
def _resolve_row_mirrors(self, state: _SlotMappingStaging) -> None:
"""Compute the final row mirrors from the staged GPU slot mapping."""
self._set_row_mirrors(
_select_written_row_mirrors(
state.candidates,
state.slots[: state.num_tokens].numpy(),
state.source_index,
)
)
state.num_tokens = 0
def _copy_host_blocks(
self,
host_block_copies: Sequence[KVCacheBlockCopy],
previous_host_write_event: torch.Event,
) -> None:
if not host_block_copies:
return
self._completed_host_copy_dst_ids.extend(
copy.dst_block_id for copy in host_block_copies
)
if self.shared_host_region is None or get_tensor_model_parallel_rank() == 0:
if self.host_caches:
previous_host_write_event.synchronize()
copy_kv_cache_blocks_inplace(
self.host_caches,
self.host_num_blocks,
host_block_copies,
)
if self.shared_host_region is not None:
get_tp_group().barrier()
def invalidate_blocks(
self, block_ids: list[int], request_state_indices: torch.Tensor
) -> None:
"""Invalidate recycled host slots in this worker's leader runtimes."""
if not block_ids:
return
device = self.cache_handles[0].runtime.device
staging = torch.tensor(block_ids, dtype=torch.int32, pin_memory=True)
blocks = staging.to(device, dtype=torch.int32, non_blocking=True)
offsets = torch.arange(self.kernel_block_size, dtype=torch.int32, device=device)
slots = (blocks[:, None] * self.kernel_block_size + offsets[None, :]).flatten()
sorted_slots = torch.sort(slots).values
state_indices = request_state_indices.to(device=device, dtype=torch.long)
for runtime in self.leader_runtimes:
runtime.invalidate_sorted_slots(sorted_slots, state_indices)
def reset_hot_state(self) -> None:
for runtime in self.leader_runtimes:
runtime.reset_hot_state()
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
stats = None
if self._metrics_pending and self._metrics_event.query():
stats = HiSparseKVConnectorStats()
for runtime in self.leader_runtimes:
group = runtime.index_group
hits, misses = group.swap_stats_host.tolist()
if hits or misses:
stats.record_snapshot(hits, misses, misses * group.stats_row_bytes)
self._metrics_pending = False
if stats.is_empty():
stats = None
self._metrics_calls += 1
if (
self._metrics_calls % _METRICS_INTERVAL == 0
and not self._metrics_pending
and not torch.cuda.is_current_stream_capturing()
):
compute_stream = current_stream()
for runtime in self.leader_runtimes:
group = runtime.index_group
compute_stream.wait_stream(group.copy_stream)
group.swap_stats_host.copy_(group.swap_stats, non_blocking=True)
group.swap_stats.zero_()
group.copy_stream.wait_stream(compute_stream)
self._metrics_event.record()
self._metrics_pending = True
return stats
def _release_completed_dma_descriptors(self) -> None:
pending = self._pending_dma_descriptors
while pending and pending[0][0].query():
_, descriptors = pending.popleft()
self._dma_free_descriptors.append(descriptors)
def _acquire_dma_descriptors(self, size: int) -> _DMADescriptors:
for index, descriptors in enumerate(self._dma_free_descriptors):
if descriptors.src.numel() >= size:
return self._dma_free_descriptors.pop(index)
return _allocate_dma_descriptors(size)
def _submit_dma_descriptors(
self,
descriptors: _DMADescriptors,
descriptor_count: int,
transfer_ids: tuple[int, ...] = (),
ready_event: torch.Event | None = None,
) -> None:
stream = self.dma_stream
assert stream is not None
if ready_event is None:
stream.wait_stream(current_stream())
else:
stream.wait_event(ready_event)
completion_event = torch.Event()
with torch.cuda.stream(stream):
ops.swap_blocks_batch(
descriptors.src[:descriptor_count],
descriptors.dst[:descriptor_count],
descriptors.sizes[:descriptor_count],
)
self.host_write_event.record(stream)
completion_event.record(stream)
self._pending_dma_descriptors.append((completion_event, descriptors))
self._dma_submitted = True
if transfer_ids:
self._pending_transfer_events.append((completion_event, transfer_ids))
self._enqueued_transfer_ids.extend(transfer_ids)
def _set_row_mirrors(self, mirrors: tuple[SparseKVRowMirror, ...]) -> None:
self._row_mirrors = mirrors
self._row_mirror_destination_starts = np.fromiter(
(mirror.destination_start for mirror in mirrors),
dtype=np.int64,
count=len(mirrors),
)
self._row_mirror_counts = np.fromiter(
(mirror.num_rows for mirror in mirrors),
dtype=np.int64,
count=len(mirrors),
)
if mirrors:
self._row_mirror_source_starts = np.asarray(
[mirror.source_starts for mirror in mirrors], dtype=np.int64
)
if self._row_mirror_source_starts.ndim != 2:
raise RuntimeError("HiSparse DMA source mappings must be rectangular.")
else:
self._row_mirror_source_starts = np.empty((0, 0), dtype=np.int64)
self._row_mirror_num_rows = int(self._row_mirror_counts.sum())
def _enqueue_row_dma(
self, layer_indices: Sequence[int], ready_event: torch.Event | None = None
) -> None:
if (
not layer_indices
or not self._row_mirror_num_rows
or not self.is_host_writer
):
return
mirrors = self._row_mirrors
num_layers = len(layer_indices)
descriptor_count = len(mirrors) * num_layers
descriptors = self._acquire_dma_descriptors(descriptor_count)
destination_starts = self._row_mirror_destination_starts
row_counts = self._row_mirror_counts
for descriptor_offset, layer_index in enumerate(layer_indices):
cache = self.cache_handles[layer_index]
source_index = cache.runtime.resident_source_index
if source_index >= self._row_mirror_source_starts.shape[1]:
raise RuntimeError("HiSparse row DMA source index is out of range.")
source_rows = self._row_mirror_source_starts[:, source_index]
source = self.resident_caches[layer_index]
destination = self.host_caches[layer_index]
row_bytes = source.shape[-1] * source.element_size()
if (
source.stride(1) * source.element_size() != row_bytes
or destination.shape[1] * destination.element_size() != row_bytes
):
raise RuntimeError("HiSparse row DMA requires contiguous rows.")
source_blocks, source_row_offsets = np.divmod(
source_rows, self.kernel_block_size
)
source_out_of_range = (
np.any(source_blocks < 0)
or np.any(source_blocks >= source.shape[0])
or np.any(source_row_offsets + row_counts > self.kernel_block_size)
)
if source_out_of_range:
raise RuntimeError("HiSparse row DMA source is out of range.")
if np.any(destination_starts < 0) or np.any(
destination_starts + row_counts > destination.shape[0]
):
raise RuntimeError("HiSparse row DMA index is out of range.")
descriptor_slice = slice(descriptor_offset, descriptor_count, num_layers)
descriptors.src_np[descriptor_slice] = (
source.data_ptr()
+ source_blocks * source.stride(0) * source.element_size()
+ source_row_offsets * row_bytes
)
descriptors.dst_np[descriptor_slice] = (
destination.data_ptr() + destination_starts * row_bytes
)
descriptors.sizes_np[descriptor_slice] = row_counts * row_bytes
self._submit_dma_descriptors(
descriptors, descriptor_count, ready_event=ready_event
)
def _enqueue_layer_mirror(self, layer_index: int) -> None:
handle = self.cache_handles[layer_index]
if not handle.host_mirror_required:
return
if layer_index in self._per_layer_mirrored:
raise RuntimeError(f"HiSparse layer {layer_index} mirrored twice.")
state = self._slot_mapping_staging
if state is not None and state.num_tokens:
# The staging stream waited on the compute stream before the
# forward launched, so this only drains pre-forward work that is
# already queued ahead of the layer kernels: no GPU bubble.
state.event.synchronize()
self._resolve_row_mirrors(state)
self._per_layer_mirrored.add(layer_index)
next_layer = layer_index + 1
if (
next_layer < len(self.cache_handles)
and self.cache_handles[next_layer].runtime.resident_source_index
== handle.runtime.resident_source_index
):
return
ready_event = self._layer_ready_events[layer_index]
ready_event.record()
pending_layers = tuple(
sorted(self._per_layer_mirrored - self._submitted_mirror_layers)
)
self._enqueue_row_dma(
pending_layers,
ready_event=ready_event,
)
self._submitted_mirror_layers.update(pending_layers)
def _record_transfer_completion(
self,
transfers: list[SparseKVPageTransfer],
*,
stream: torch.Stream | None = None,
) -> None:
if not transfers:
return
if stream is None:
if not self.is_host_writer:
return
stream = self.dma_stream
assert stream is not None
completion_event = torch.Event()
completion_event.record(stream)
transfer_ids = tuple(transfer.transfer_id for transfer in transfers)
self._pending_transfer_events.append((completion_event, transfer_ids))
self._enqueued_transfer_ids.extend(transfer_ids)
def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None:
"""Restore imported tails on every rank before its forward or graph replay."""
if not transfers:
return
for layer_index, cache in enumerate(self.cache_handles):
source_index = cache.runtime.resident_source_index
source = self.host_caches[layer_index]
destination = self.resident_caches[layer_index]
for transfer in transfers:
host_start = transfer.host_block_id * self.kernel_block_size
resident_block = transfer.resident_block_ids[source_index]
# Each import pays one H2D page copy and enqueue per layer/rank
# instead of tail prefill on D. Restore the prompt rows before
# decode appends to this page; unused rows do not extend the
# logical sequence.
# cudaHostRegister pins this pool outside PyTorch's allocator,
# so the sync checker cannot recognize it via is_pinned().
with gpu_sync_allowed():
destination[resident_block].copy_(
source[host_start : host_start + self.kernel_block_size],
non_blocking=True,
)
self._record_transfer_completion(transfers, stream=current_stream())
def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
if self.cache_handles[0].runtime.eager_host_mirror:
self._record_transfer_completion(transfers)
else:
self._enqueue_transfers(transfers)
def _enqueue_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
if not transfers or not self.is_host_writer:
return
num_layers = len(self.cache_handles)
descriptor_count = len(transfers) * num_layers
descriptors = self._acquire_dma_descriptors(descriptor_count)
destination_rows = np.fromiter(
(transfer.host_block_id * self.kernel_block_size for transfer in transfers),
dtype=np.int64,
count=len(transfers),
)
source_blocks_by_transfer = np.asarray(
[transfer.resident_block_ids for transfer in transfers], dtype=np.int64
)
if source_blocks_by_transfer.ndim != 2:
raise RuntimeError(
"HiSparse spill DMA source mappings must be rectangular."
)
for layer_index, cache in enumerate(self.cache_handles):
source_index = cache.runtime.resident_source_index
if source_index >= source_blocks_by_transfer.shape[1]:
raise RuntimeError("HiSparse spill DMA source index is out of range.")
source_blocks = source_blocks_by_transfer[:, source_index]
source = self.resident_caches[layer_index]
destination = self.host_caches[layer_index]
if np.any(source_blocks < 0) or np.any(source_blocks >= source.shape[0]):
raise RuntimeError("HiSparse spill DMA source is out of range.")
if np.any(destination_rows < 0) or np.any(
destination_rows + self.kernel_block_size > destination.shape[0]
):
raise RuntimeError("HiSparse spill DMA destination is out of range.")
row_bytes = source.shape[-1] * source.element_size()
descriptor_slice = slice(layer_index, descriptor_count, num_layers)
descriptors.src_np[descriptor_slice] = (
source.data_ptr()
+ source_blocks * source.stride(0) * source.element_size()
)
descriptors.dst_np[descriptor_slice] = (
destination.data_ptr() + destination_rows * row_bytes
)
descriptors.sizes_np[descriptor_slice] = self.kernel_block_size * row_bytes
self._submit_dma_descriptors(
descriptors,
descriptor_count,
transfer_ids=tuple(transfer.transfer_id for transfer in transfers),
)
def _enqueue_host_mirror(
self,
ready_event: torch.Event | None = None,
) -> None:
active = [
(index, handle)
for index, handle in enumerate(self.cache_handles)
if handle.num_actual_tokens != 0
]
if not active:
return
def mirror_key(handle: HiSparseCacheHandle) -> tuple:
slots = handle.mirror_slot_mapping
return (
handle.num_actual_tokens,
handle.num_decode_tokens,
handle.decode_batch,
handle.host_mirror_required,
handle.runtime.eager_host_mirror,
None if slots is None else slots.data_ptr(),
)
keys = {mirror_key(handle) for _, handle in active}
if len(keys) > 1:
raise RuntimeError(
f"HiSparse cache layers disagree on mirror metadata: {list(keys)}."
)
cache = active[0][1]
if not cache.host_mirror_required:
return
dst_slots = cache.mirror_slot_mapping
if dst_slots is None:
raise RuntimeError("HiSparse host mirror has no source slot mapping.")
num_rows = min(cache.num_actual_tokens, dst_slots.numel())
if num_rows == 0:
return
if self.is_host_writer:
# The drafter has not written its layers' rows yet; the next
# start_step mirrors them after it runs.
self._draft_mirror_pending = bool(self._draft_layers)
expected_layers = {index for index, _ in active}.difference(
self._draft_layers
)
if self._per_layer_mirrored and self._per_layer_mirrored != expected_layers:
raise RuntimeError(
"HiSparse per-layer DMA did not mirror every active layer: "
f"expected {sorted(expected_layers)}, got "
f"{sorted(self._per_layer_mirrored)}."
)
pending_layers = tuple(
sorted(expected_layers - self._submitted_mirror_layers)
)
if pending_layers:
self._enqueue_row_dma(pending_layers, ready_event=ready_event)
self._submitted_mirror_layers.update(pending_layers)
assert cache.req_id_per_token is not None
for _, handle in active:
if handle.runtime.is_group_leader:
handle.runtime.invalidate_written_slots(
dst_slots[:num_rows],
cache.req_id_per_token[:num_rows],
)
def _finish_mirror_phase(self, ready_event: torch.Event | None = None) -> None:
state = self._slot_mapping_staging
if state is not None and state.num_tokens:
# Measured free: blocking here costs no throughput (26.7 vs 27.6
# gen tok/s) and lets every step mirror exactly the written rows.
state.event.synchronize()
self._resolve_row_mirrors(state)
self._enqueue_host_mirror(ready_event)
self._clear_forward_mirror_state()
def finish_forward(self) -> None:
compute_stream = current_stream()
self._forward_ready_event.record()
self._finish_mirror_phase(self._forward_ready_event)
if self.is_host_writer:
if self._dma_submitted:
compute_stream.wait_event(self.host_write_event)
self._dma_submitted = False
else:
self.host_write_event.record(compute_stream)
self._release_completed_dma_descriptors()
def _finish_previous_step(self) -> None:
"""Mirror the previous step's draft-layer rows, then hand its pages over.
Runs before this step's work, so after the previous step's drafter.
Post-forward transfers are ordered behind the draft mirror, so a page is
never reported clean, and its resident block never released, before
its draft-layer rows reach the host. Both land in the previous step's
host write event, which this step waits on before its forward and its
host copies, since the scheduler may already have reused the
finished requests' blocks they read and write.
"""
if self._draft_mirror_pending:
self._draft_mirror_pending = False
self._enqueue_row_dma(self._draft_layers)
transfers = self._post_forward_transfers
self._post_forward_transfers = []
self._submit_transfers(transfers)
def take_completed_host_copies(self) -> list[int]:
"""Drain host copies this worker has enqueued for this step."""
completed = self._completed_host_copy_dst_ids
self._completed_host_copy_dst_ids = []
return completed
def take_transfer_updates(self) -> tuple[list[int], list[int]]:
enqueued = self._enqueued_transfer_ids
self._enqueued_transfer_ids = []
completed: list[int] = []
while self._pending_transfer_events:
event, transfer_ids = self._pending_transfer_events[0]
if not event.query():
break
self._pending_transfer_events.popleft()
completed.extend(transfer_ids)
return enqueued, completed
def shutdown(self) -> None:
if not self._initialized:
return
if self._slot_mapping_staging is not None:
self._slot_mapping_staging.stream.synchronize()
if self.dma_stream is not None:
self.dma_stream.synchronize()
release_pinned_state(
[cache.runtime for cache in self.cache_handles],
self.pinned_host_pools,
self.shared_host_region,
)
drop_hisparse_host_pool(self.vllm_config)
self._initialized = False
@@ -1,656 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import math
import os
import threading
import time
from collections.abc import Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, NamedTuple
import msgspec
import regex as re
import torch
import zmq
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.utils.network_utils import (
get_ip,
get_open_port,
make_zmq_socket,
)
if TYPE_CHECKING:
pass
from dataclasses import field
from enum import Enum
logger = init_logger(__name__)
Transfer = tuple[int, float]
EngineId = str
ReqId = str
TransferId = str
TransferOffsetsKey = tuple[str, tuple[int, ...], tuple[int, ...], torch.dtype]
def _positive_finite_timeout(name: str, value: Any) -> float:
timeout = float(value)
if not math.isfinite(timeout) or timeout <= 0:
raise ValueError(f"{name} must be finite and greater than zero")
return timeout
class MoRIIOTransferAck(NamedTuple):
transfer_id: TransferId
consumer_tp_size: int = 1
@dataclass
class WriteTask:
request_id: ReqId
transfer_id: TransferId
dst_engine_id: str
local_block_ids: BlockIds
remote_block_ids_hint: BlockIds | None
layer_name: str
event: torch.cuda.Event
remote_notify_port: int
remote_ip: str
remote_tp_size: int | None = None
multi_pod_hosts: list[str] = field(default_factory=list)
remote_dp_size_local: int = 0
remote_dp_size: int = 1
enqueue_time: float = field(default_factory=time.perf_counter)
retried: int = 0
@dataclass
class LayerTransferPlan:
"""Plan for transferring a single layer."""
request_id: ReqId
transfer_id: TransferId
layer_name: str
sess_idx: int
transfer_local_offsets: list[int]
transfer_remote_offsets: list[int]
transfer_sizes: list[int]
use_batch: bool = True
@dataclass
class RemoteAllocInfo:
"""Information about remote block allocation."""
block_ids: BlockIds | None
writes_done: int = 0
writes_expected: int | None = None
decode_dp_rank: int = 0
completion_request_id: str | None = None
completion_notify_port: int | None = None
completion_remote_ip: str | None = None
completion_notified: bool = False
transfer_statuses: list[Any] = field(default_factory=list)
transfer_offsets: dict[
TransferOffsetsKey,
tuple[list[int], list[int], list[int]],
] = field(default_factory=dict)
class ROLE(Enum):
PRODUCER = "producer"
CONSUMER = "consumer"
NOTINIT = "notinit"
class MoRIIOAgentMetadata(
msgspec.Struct,
omit_defaults=True, # type: ignore[call-arg]
# required for @cached_property.d
dict=True,
):
engine_id: str
agent_metadata: bytes
kv_caches_base_addr: list[int]
num_blocks: int
block_len: int
attn_backend_name: str
class RoleManager:
"""Manages role state across the connector."""
_instance: "RoleManager | None" = None
_lock = threading.Lock()
def __init__(self) -> None:
self._role: ROLE = ROLE.NOTINIT
@classmethod
def get_instance(cls) -> "RoleManager":
if cls._instance is None:
with cls._lock:
if cls._instance is None:
cls._instance = cls()
return cls._instance
def set_role(self, role: ROLE) -> None:
"""Set the current role."""
with self._lock:
self._role = role
def get_role(self) -> ROLE:
"""Get the current role."""
return self._role
def set_role(role: ROLE):
"""Set the global role."""
RoleManager.get_instance().set_role(role)
def get_role() -> ROLE:
"""Get the global role."""
return RoleManager.get_instance().get_role()
class MoRIIOMode(Enum):
READ = "read"
WRITE = "write"
class TransferBatchState(Enum):
"""Verdict for a group of transfer statuses that belong to one request.
A request's KV transfer is spread over one status per layer (two for a KDA
layer), so a single status never decides the request: PENDING means at
least one is still in flight and none has failed, FAILED means at least
one failed, DONE means all succeeded.
"""
DONE = "done"
FAILED = "failed"
PENDING = "pending"
class MoRIIOError(Exception):
"""Base exception for MoRIIO operations."""
pass
class HandshakeError(MoRIIOError):
"""Exception raised when handshake fails."""
pass
class TransferError(MoRIIOError):
"""Exception raised when transfer fails."""
pass
def get_moriio_mode(kv_transfer_config: KVTransferConfig) -> MoRIIOMode:
read_mode = str(
kv_transfer_config.kv_connector_extra_config.get("read_mode", "false")
).lower().strip() in ("true", "1")
logger.debug("MoRIIO Connector read_mode: %s", read_mode)
if read_mode:
return MoRIIOMode.READ
else:
return MoRIIOMode.WRITE
def get_port_offset(dp_rank: int, tp_rank: int, tp_size: int = 1) -> int:
return (dp_rank) * tp_size + tp_rank
def fold_local_rank(global_dp_rank: int, dp_size_local: int) -> int:
"""Fold a global DP rank into its pod-local rank [0, dp_size_local).
``dp_size_local == 0`` is the external-DP sentinel (local size unknown):
return the rank unchanged, since a global DP rank is always < the global
DP size so no folding is needed and the modulo is skipped (never divides
by zero).
"""
return global_dp_rank % dp_size_local if dp_size_local else global_dp_rank
def pod_index(global_dp_rank: int, dp_size_local: int) -> int:
"""Pod index (0-based) a global DP rank lives on for Wide-EP multi-pod.
``dp_size_local == 0`` (external-DP sentinel) collapses to a single pod.
"""
return global_dp_rank // dp_size_local if dp_size_local else 0
def resolve_host_ip(extra_config: dict) -> str:
"""The IP this MoRIIO process advertises for KV transfer.
Honors an explicit ``host_ip`` in ``kv_connector_extra_config`` before
falling back to ``get_ip()``. An external router/orchestrator can set it to
the node's routable address; this is required under frameworks (e.g. Ray)
where ``get_ip()`` resolves to an unroutable public IP and ``VLLM_HOST_IP``
cannot be propagated to the worker processes that bind the transfer engine.
"""
return extra_config.get("host_ip") or get_ip()
_DEPRECATED_ENV_VARS: dict[str, str] = {
"VLLM_MORIIO_CONNECTOR_READ_MODE": "read_mode",
"VLLM_MORIIO_QP_PER_TRANSFER": "qp_per_transfer",
"VLLM_MORIIO_POST_BATCH_SIZE": "post_batch_size",
"VLLM_MORIIO_NUM_WORKERS": "num_workers",
}
def _warn_deprecated_env_vars() -> None:
for env_var, new_key in _DEPRECATED_ENV_VARS.items():
if env_var in os.environ:
logger.warning_once(
"The environment variable %s is deprecated and ignored. "
"Set %r inside kv_transfer_config.kv_connector_extra_config "
"instead.",
env_var,
new_key,
)
@dataclass
class MoRIIOConfig:
local_ip: str
local_kv_port: int
proxy_ip: str
local_ping_port: int
proxy_ping_port: int
http_port: int
handshake_port: int
notify_port: int
tp_rank: int
dp_rank: int
dp_size: int
tp_size: int
transfer_timeout: float
defer_timeout: float
recv_abort_timeout: float
read_mode: bool = False
qp_per_transfer: int = 1
post_batch_size: int = -1
num_workers: int = 1
backend: str = "rdma"
@classmethod
def from_vllm_config(cls, vllm_config: VllmConfig) -> "MoRIIOConfig":
# Port Configuration:
# local_ping_port -> Outgoing heartbeat to proxy
# proxy_ping_port -> Remote proxy's heartbeat ingress port
# http_port -> Instance's HTTP service endpoint
# local_kv_port -> service port for mori engine
# notify_port -> For synchronizing stages between prefill and decode
# handshake_port -> For initial handshake between mori engine
# Optional tuning knobs
# read_mode -> If true, run the connector in READ mode (consumer
# pulls KV from producer) instead of the default
# WRITE mode.
# transfer_timeout -> Timeout for waiting_for_transfer_complete before
# raising TransferError (sec).
# defer_timeout -> Timeout for deferred WRITE allocation or a missing
# READ release notification (sec).
# recv_abort_timeout -> Timeout before an in-flight recv whose RDMA
# completion never arrived is aborted (sec).
# Knobs for RDMA transfers, ignored if on xgmi backend
# qp_per_transfer -> Number of RDMA Queue Pairs per KV transfer.
# post_batch_size -> Batch size for posting transfer work requests
# (-1 lets the MoRI backend choose).
# num_workers -> Number of background worker threads the MoRI
# engine uses for transfer processing.
# TODO : merge notify_port and handshake_port to simplify port management
# supports non-contiguous ports
assert vllm_config.kv_transfer_config is not None, (
"kv_transfer_config must be set for MoRIIOConnector"
)
_warn_deprecated_env_vars()
kv_transfer_config = vllm_config.kv_transfer_config
extra_config = kv_transfer_config.kv_connector_extra_config
tp_rank = get_tensor_model_parallel_rank()
# Fold the global data_parallel_rank back to [0, dp_size_local) for
# per-node port allocation (handles the external-DP sentinel).
pc = vllm_config.parallel_config
dp_rank = fold_local_rank(pc.data_parallel_rank, pc.data_parallel_size_local)
base_notify_port = int(extra_config["notify_port"])
tp_size = get_tensor_model_parallel_world_size()
port_offset = get_port_offset(dp_rank, tp_rank)
backend = str(extra_config.get("backend", "rdma")).lower()
if backend not in ("rdma", "xgmi"):
raise ValueError(
f"Invalid MoRIIO backend {backend!r} in kv_connector_extra_config; "
"must be one of 'rdma' or 'xgmi'."
)
transfer_timeout = _positive_finite_timeout(
"kv_connector_extra_config.transfer_timeout",
extra_config.get(
"transfer_timeout", MoRIIOConstants.DEFAULT_TRANSFER_TIMEOUT
),
)
defer_timeout = _positive_finite_timeout(
"kv_connector_extra_config.defer_timeout",
extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT),
)
legacy_recv_abort_timeout = os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S")
if legacy_recv_abort_timeout is not None:
logger.warning_once(
"The environment variable VLLM_MORIIO_TRANSFER_TIMEOUT_S is "
"deprecated. Set 'recv_abort_timeout' inside "
"kv_transfer_config.kv_connector_extra_config instead."
)
recv_abort_timeout = _positive_finite_timeout(
"kv_connector_extra_config.recv_abort_timeout",
extra_config.get(
"recv_abort_timeout",
legacy_recv_abort_timeout or MoRIIOConstants.DEFAULT_RECV_ABORT_TIMEOUT,
),
)
return cls(
local_ip=resolve_host_ip(extra_config),
local_kv_port=get_open_port(),
proxy_ip=extra_config.get("proxy_ip", ""),
local_ping_port=get_open_port(),
proxy_ping_port=int(extra_config.get("proxy_ping_port", 0)),
http_port=int(extra_config["http_port"]),
handshake_port=int(extra_config["handshake_port"]),
notify_port=base_notify_port + port_offset,
tp_rank=tp_rank,
dp_rank=dp_rank,
# Advertise the GLOBAL DP world size to the routing proxy so it can
# address every rank across pods; dp_size_local above is only for
# per-pod port folding, not for routing.
dp_size=pc.data_parallel_size,
tp_size=tp_size,
read_mode=get_moriio_mode(kv_transfer_config) == MoRIIOMode.READ,
qp_per_transfer=int(extra_config.get("qp_per_transfer", 1)),
post_batch_size=int(extra_config.get("post_batch_size", -1)),
num_workers=int(extra_config.get("num_workers", 1)),
backend=backend,
transfer_timeout=transfer_timeout,
defer_timeout=defer_timeout,
recv_abort_timeout=recv_abort_timeout,
)
class MoRIIOConstants:
"""Constants for MoRIIO connector."""
# ZMQ message types
GET_META_MSG = b"get_meta_msg"
POP_DONE_RECV = b"pop_done_recv"
OVER = b"OVER"
COMPLETION_PREFIX = "cmpl"
TRANSFER_PREFIX = "tx"
PING_INTERVAL = 3
MAX_PING_RETRIES = 100
DEFAULT_HANDSHAKE_PORT = "6301"
DEFAULT_NOTIFY_PORT = "61005"
VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT = 3600
# Timeout (seconds) for waiting_for_transfer_complete before raising TransferError.
# Overridable via kv_connector_extra_config["transfer_timeout"].
DEFAULT_TRANSFER_TIMEOUT = 30.0
# Timeout (seconds) for deferred WRITE allocation or a missing READ release
# notification.
# Overridable via kv_connector_extra_config["defer_timeout"].
DEFAULT_DEFER_TIMEOUT = 60.0
# Timeout (seconds) before an in-flight recv whose RDMA completion was lost
# is aborted, so the decode worker does not hang on it forever.
# Overridable via kv_connector_extra_config["recv_abort_timeout"].
DEFAULT_RECV_ABORT_TIMEOUT = 120.0
# The router embeds both zmq_addresses in the request_id:
# "___prefill_addr_{zmq}___decode_addr_{zmq}_{32-hex-uuid}"
# MoRIIO zmq_address format: "host:IP,handshake:PORT,notify:PORT"
#
# This lets each connector side parse the peer's connection info without
# requiring the router to pass it explicitly in kv_transfer_params.
_PREFILL_ZMQ_RE = re.compile(r"___prefill_addr_(.+?)___decode_addr_")
# vLLM wraps the router's X-Request-Id as "cmpl-<id>-<seq>-<hex>" so there may
# be a trailing "-<seq>-<hex>" suffix after the 32-char UUID. Allow it.
_DECODE_ZMQ_RE = re.compile(r"___decode_addr_(.+)_[0-9a-f]{32}(?:-.*)?$")
def parse_moriio_zmq_address(
zmq_address: str,
) -> tuple[str, int, int]:
"""Parse the MoRI-IO zmq address into its components.
Parses ``"host:IP,handshake:PORT,notify:PORT"`` into
(host, handshake_port, notify_port).
Each key-value pair is split on the *first* colon so that IPv6 addresses
(e.g. ``host:::1``) are handled correctly. Raises ``ValueError`` if any
of ``host``, ``handshake``, or ``notify`` keys are absent or if the port
values are non-numeric.
"""
parts: dict[str, str] = {}
for segment in zmq_address.split(","):
key, _, val = segment.partition(":")
parts[key.strip()] = val.strip()
try:
host = parts["host"]
handshake_port = int(parts["handshake"])
notify_port = int(parts["notify"])
except (KeyError, ValueError) as e:
raise ValueError(
f"Malformed zmq_address {zmq_address!r}: expected "
f"'host:IP,handshake:PORT,notify:PORT' format"
) from e
return host, handshake_port, notify_port
def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str | None:
"""Extract the *peer's* zmq_address from the vLLM router request_id.
The producer (prefill) needs the decode's address; the consumer (decode)
needs the prefill's address.
Returns ``None`` when the request_id does not encode peer info. The
llm-d routing sidecar (``llm-d-inference-scheduler``) does not embed
addresses in ``request_id``; instead it passes ``remote_host``,
``remote_handshake_port`` and ``remote_notify_port`` explicitly in
``kv_transfer_params``. Callers must handle the ``None`` return by
falling back to those fields. See ``add_new_req`` for the canonical
fallback path.
"""
if is_producer:
m = _DECODE_ZMQ_RE.search(request_id)
else:
m = _PREFILL_ZMQ_RE.search(request_id)
if m is None:
return None
return m.group(1)
@dataclass
class ReqMeta:
"""Metadata for a single request."""
transfer_id: TransferId
local_block_ids: BlockIds
remote_block_ids: BlockIds
remote_host: str
remote_port: int
remote_handshake_port: int
remote_notify_port: int
remote_engine_id: str
tp_size: int
remote_dp_size: int
# Prefill DP rank that owns this request's KV (forwarded by the proxy). The
# read must target this rank's memory registration; the default 0 preserves
# the symmetric single-DP behaviour.
remote_dp_rank: int = 0
# Multi-pod: list of remote pod IPs indexed by pod_idx.
multi_pod_hosts: list[str] = field(default_factory=list)
# Per-pod DP size; 0 means fallback to remote_dp_size.
remote_dp_size_local: int = 0
class MoRIIOConnectorMetadata(KVConnectorMetadata):
def __init__(self):
self.reqs_to_recv: dict[ReqId, ReqMeta] = {}
self.reqs_to_save: dict[ReqId, ReqMeta] = {}
self.reqs_to_send: dict[ReqId, float] = {}
self.transfer_id_to_request_id: dict[TransferId, ReqId] = {}
def __repr__(self):
return (
f"MoRIIOConnectorMetadata: reqs_to_recv={self.reqs_to_recv}, "
f"reqs_to_save={self.reqs_to_save}, "
f"reqs_to_send={self.reqs_to_send}, "
f"transfer_id_to_request_id={self.transfer_id_to_request_id}"
)
def add_new_req(
self,
request_id: ReqId,
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
write_mode=False,
):
"""Ingest a peer's ``kv_transfer_params`` into a typed ``ReqMeta``.
This is the single place peer info enters the connector. The
``kv_transfer_params`` contract (as produced by the llm-d sidecar /
vLLM router, or echoed by the prefill leg's ``request_finished``):
Required (always):
* ``transfer_id`` -- stable id shared by both legs.
* ``remote_engine_id`` -- peer engine id for the handshake table.
* ``remote_block_ids`` -- peer block ids (may be [] in WRITE mode,
where decode allocates its own blocks).
Peer address -- ONE of the following two must resolve:
* embedded in ``request_id`` (vLLM-router PD id form), OR
* explicit ``remote_host`` + ``remote_handshake_port`` +
``remote_notify_port`` (llm-d sidecar / returnable path).
If neither resolves we raise -- there is no safe default host/port.
Optional (defaulted):
* ``tp_size`` (default 1) -- peer TP size.
* ``remote_dp_size`` (default 1) -- peer GLOBAL DP size.
* ``remote_dp_size_local`` (default = ``remote_dp_size``) -- per-pod
DP size for Wide-EP multi-pod port/host
folding; 0/absent means single-pod.
* ``remote_hosts`` (default [remote_host]) -- per-pod IP list
indexed by ``pod_idx = rank // dp_local``.
Routing keys consumed elsewhere (NOT here): ``remote_dp_rank`` /
``remote_dp_rank_override`` gate the decode->prefill notify target in
MoRIIOConnectorScheduler; they are router-authoritative and never
self-derived (see that class's request routing contract).
"""
transfer_id = kv_transfer_params["transfer_id"]
# Try request_id embedded address first, fallback to explicit params.
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
if peer_zmq is not None:
remote_host, remote_handshake_port, remote_notify_port = (
parse_moriio_zmq_address(peer_zmq)
)
else:
try:
remote_host = kv_transfer_params["remote_host"]
if not remote_host:
raise ValueError(
f"request_id {request_id!r} does not embed a peer "
f"zmq_address and kv_transfer_params['remote_host'] is "
f"empty; cannot route MoRI-IO transfer"
)
remote_handshake_port = int(kv_transfer_params["remote_handshake_port"])
remote_notify_port = int(kv_transfer_params["remote_notify_port"])
except (KeyError, TypeError, ValueError) as e:
raise ValueError(
f"request_id {request_id!r} does not embed a peer "
f"zmq_address and kv_transfer_params is missing one or "
f"more sidecar-fallback keys (need remote_host, "
f"remote_handshake_port, remote_notify_port): {e}"
) from e
# Multi-pod: use multi_pod_hosts list or fallback to single host.
_pod_hosts = kv_transfer_params.get("remote_hosts") or [remote_host]
if not isinstance(_pod_hosts, list):
_pod_hosts = [_pod_hosts]
_pod_hosts = [str(h) for h in _pod_hosts]
_remote_dp_size_local = int(
kv_transfer_params.get(
"remote_dp_size_local",
kv_transfer_params.get("remote_dp_size", 1),
)
)
_req = ReqMeta(
transfer_id=transfer_id,
local_block_ids=local_block_ids,
remote_block_ids=kv_transfer_params["remote_block_ids"],
remote_engine_id=kv_transfer_params["remote_engine_id"],
remote_host=remote_host,
remote_port=int(remote_handshake_port),
remote_handshake_port=int(remote_handshake_port),
remote_notify_port=int(remote_notify_port),
# Remote peer TP degree (used as remote_tp_size downstream). The
# decode side must know the prefiller's TP degree to map producer
# ranks onto the correct decode rank (and vice versa). The old plain
# "tp_size" key (#46332) is absent on WRITE producer requests and
# defaulted to 1, collapsing every producer rank onto decode rank 0
# and corrupting the transfer; prefer "remote_tp_size". 0 == unknown
# (treated as homogeneous downstream).
tp_size=int(
kv_transfer_params.get("remote_tp_size")
or kv_transfer_params.get("tp_size")
or 0
),
remote_dp_size=kv_transfer_params.get("remote_dp_size", 1),
remote_dp_rank=kv_transfer_params.get("remote_dp_rank", 0),
multi_pod_hosts=_pod_hosts,
remote_dp_size_local=_remote_dp_size_local,
)
if write_mode:
self.reqs_to_save[request_id] = _req
else:
self.reqs_to_recv[request_id] = _req
@contextlib.contextmanager
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
"""Context manager for a ZMQ socket."""
if socket_type not in (zmq.ROUTER, zmq.REQ, zmq.DEALER):
raise ValueError(f"Unexpected socket type: {socket_type}")
ctx: zmq.Context | None = None
try:
ctx = zmq.Context() # type: ignore[attr-defined]
yield make_zmq_socket(
ctx=ctx, path=addr, socket_type=socket_type, bind=socket_type == zmq.ROUTER
)
finally:
if ctx is not None:
ctx.destroy(linger=0)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,96 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Keep discovery heartbeats independent of the model worker's GIL."""
import json
import logging
import select
import subprocess
import sys
from typing import Any
import msgpack
import zmq
logger = logging.getLogger(__name__)
class MoRIIOHeartbeat:
def __init__(
self,
endpoint: str,
payload: dict[str, Any],
interval: float,
max_retries: int = 100,
):
self._process = subprocess.Popen(
[sys.executable, __file__, endpoint, str(interval), str(max_retries)],
stdin=subprocess.PIPE,
text=True,
)
assert self._process.stdin is not None
try:
self._process.stdin.write(json.dumps(payload) + "\n")
self._process.stdin.flush()
except BaseException:
self.shutdown()
raise
def check_health(self) -> None:
returncode = self._process.poll()
if returncode is not None:
raise RuntimeError(
f"MoRIIO discovery heartbeat exited with code {returncode}"
)
def shutdown(self) -> None:
assert self._process.stdin is not None
try:
if not self._process.stdin.closed:
self._process.stdin.close()
except BrokenPipeError:
pass
finally:
try:
self._process.wait(timeout=5)
except subprocess.TimeoutExpired:
self._process.terminate()
try:
self._process.wait(timeout=1)
except subprocess.TimeoutExpired:
self._process.kill()
self._process.wait(timeout=1)
def main() -> None:
endpoint, interval_arg, max_retries_arg = sys.argv[1:]
initial = sys.stdin.readline()
if not initial:
return
payload = msgpack.dumps(json.loads(initial))
interval = float(interval_arg)
max_retries = int(max_retries_arg)
failures = 0
with zmq.Context() as context, context.socket(zmq.DEALER) as sock:
sock.setsockopt(zmq.LINGER, 0)
sock.setsockopt(zmq.SNDTIMEO, 1000)
sock.connect(endpoint)
while not select.select([sys.stdin], [], [], 0)[0]:
try:
sock.send(payload)
failures = 0
except zmq.Again:
failures = 0
logger.warning("MoRIIO discovery heartbeat send failed; retrying")
except zmq.ZMQError:
failures += 1
logger.exception("MoRIIO discovery heartbeat send failed")
if failures >= max_retries:
raise
# EOF also stops the child when the worker exits without cleanup.
if select.select([sys.stdin], [], [], interval)[0]:
break
if __name__ == "__main__":
main()
@@ -1,615 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Mapping
from typing import NamedTuple, cast
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
MambaConvSplitInfo,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
UniformTypeKVCacheSpecs,
)
class LayerTransferGeometry(NamedTuple):
num_blocks: int
block_size: int
block_len: int
slot_size_bytes: int
block_stride: int
local_kv_stride: int | None
remote_kv_stride: int | None
transfers_per_block: int
regions_per_block: int
split_kv_regions: bool
class MambaTransferGeometry(NamedTuple):
"""Geometry of a hybrid (mamba/KDA) layer's recurrent state.
A KDA layer's packed page is exposed as ``conv_state`` and ``ssm_state``
views. Each is slot-strided, so a slot's bytes live at
``slot * slot_stride * element_size`` and span ``slot_bytes``. The two
tensors are registered as two separate MoRIIO regions per layer.
"""
conv_slot_stride: int
ssm_slot_stride: int
ssm_slot_bytes: int
conv_element_size: int
ssm_element_size: int
conv_region_len: int
ssm_region_len: int
def get_mamba_transfer_geometry(
conv: torch.Tensor,
ssm: torch.Tensor,
) -> MambaTransferGeometry:
"""Derive the slot-strided geometry of a KDA layer's conv + ssm state.
``conv``/``ssm`` are zero-copy views over the layer's packed page. Slot byte
offset is ``slot * slot_stride * element_size`` and each
slot spans ``subtensor[0].numel() * element_size`` bytes; the whole tensor
is registered as one region (``*_region_len`` bytes).
"""
if conv.ndim == 0 or ssm.ndim == 0 or conv.shape[0] == 0 or ssm.shape[0] == 0:
raise ValueError("KDA conv/ssm caches must contain at least one slot")
if conv.shape[0] != ssm.shape[0]:
raise ValueError(
f"KDA conv/ssm slot count mismatch: {conv.shape[0]} != {ssm.shape[0]}"
)
conv_slot_bytes = conv[0].numel() * conv.element_size()
ssm_slot_bytes = ssm[0].numel() * ssm.element_size()
conv_available_bytes = (
conv.untyped_storage().nbytes() - conv.storage_offset() * conv.element_size()
)
ssm_available_bytes = (
ssm.untyped_storage().nbytes() - ssm.storage_offset() * ssm.element_size()
)
if conv.shape[0]:
conv_required_bytes = (conv.shape[0] - 1) * conv.stride(
0
) * conv.element_size() + conv_slot_bytes
ssm_required_bytes = (ssm.shape[0] - 1) * ssm.stride(
0
) * ssm.element_size() + ssm_slot_bytes
if conv_required_bytes > conv_available_bytes:
raise ValueError("KDA conv slots exceed the registered storage extent")
if ssm_required_bytes > ssm_available_bytes:
raise ValueError("KDA ssm slots exceed the registered storage extent")
return MambaTransferGeometry(
conv_slot_stride=conv.stride(0),
ssm_slot_stride=ssm.stride(0),
ssm_slot_bytes=ssm_slot_bytes,
conv_element_size=conv.element_size(),
ssm_element_size=ssm.element_size(),
# Registered region must span the full slot-strided extent: stride(0)
# may exceed the per-slot element count, so numel() would under-size the
# region and truncate the highest-slot transfers. shape[0]*stride(0)
# covers every byte a ``slot * slot_stride * elem`` offset can address.
# Cap at the bytes remaining from each view's storage offset. A packed
# conv+ssm page places ssm at a non-zero intra-buffer offset,
# so do not register past the end of the shared backing allocation.
conv_region_len=min(
conv.shape[0] * conv.stride(0) * conv.element_size(),
conv_available_bytes,
),
ssm_region_len=min(
ssm.shape[0] * ssm.stride(0) * ssm.element_size(),
ssm_available_bytes,
),
)
def build_layer_to_spec(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheSpec]:
layer_to_spec: dict[str, KVCacheSpec] = {}
for group in kv_cache_config.kv_cache_groups:
group_spec = group.kv_cache_spec
if isinstance(group_spec, UniformTypeKVCacheSpecs):
layer_to_spec.update(
{
layer_name: group_spec.kv_cache_specs[layer_name]
for layer_name in group.layer_names
}
)
else:
layer_to_spec.update(
{layer_name: group_spec for layer_name in group.layer_names}
)
return layer_to_spec
def is_mla_cache_layer(
layer_to_spec: Mapping[str, KVCacheSpec], layer_name: str
) -> bool:
try:
spec = layer_to_spec[layer_name]
except KeyError as e:
raise ValueError(f"Missing KV cache spec for layer {layer_name}") from e
return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
def _spec_dim_matches(value: int, expected: int | None) -> bool:
return expected is None or value == expected
def _kernel_layout_matches(
spec: KVCacheSpec, kernel_block_size: int, num_kv_heads: int, head_dim: int
) -> bool:
if kernel_block_size <= 0 or spec.block_size % kernel_block_size != 0:
return False
return _spec_dim_matches(
num_kv_heads, getattr(spec, "num_kv_heads", None)
) and _spec_dim_matches(head_dim, getattr(spec, "head_size", None))
def _select_kernel_block_layout(
layer_name: str, shape: torch.Size, spec: KVCacheSpec
) -> tuple[int, int, int]:
axis2_matches = _kernel_layout_matches(spec, shape[2], shape[3], shape[4])
axis3_matches = _kernel_layout_matches(spec, shape[3], shape[2], shape[4])
if axis2_matches and axis3_matches and shape[2] != shape[3]:
raise ValueError(
f"Ambiguous MoRIIO kernel-block K/V cache shape for layer "
f"{layer_name}: {tuple(shape)}"
)
if axis2_matches:
return shape[2], shape[3], shape[4]
if axis3_matches:
return shape[3], shape[2], shape[4]
raise ValueError(
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
f"{tuple(shape)} does not contain block size {spec.block_size}"
)
def get_layer_transfer_geometry(
layer_name: str,
kv_cache: torch.Tensor,
layer_to_spec: Mapping[str, KVCacheSpec],
remote_num_blocks: int | None = None,
) -> LayerTransferGeometry:
spec = layer_to_spec[layer_name]
shape = kv_cache.shape
stride = kv_cache.stride()
element_size = kv_cache.element_size()
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
_, num_blocks = shape[:2]
kernel_blocks_per_block = 1
if shape[2] == spec.block_size:
block_size, num_kv_heads, head_dim = shape[2:]
elif shape[3] == spec.block_size:
num_kv_heads, block_size, head_dim = shape[2:]
else:
kernel_num_blocks = num_blocks
kernel_block_size, num_kv_heads, head_dim = _select_kernel_block_layout(
layer_name, shape, spec
)
kernel_blocks_per_block = spec.block_size // kernel_block_size
if kernel_num_blocks % kernel_blocks_per_block != 0:
raise ValueError(
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
f"not divisible by {kernel_blocks_per_block}"
)
num_blocks = kernel_num_blocks // kernel_blocks_per_block
block_size = spec.block_size
slot_size_bytes = num_kv_heads * head_dim * element_size
block_len = block_size * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[1] * kernel_blocks_per_block,
local_kv_stride=stride[0],
remote_kv_stride=(
stride[1] * kernel_blocks_per_block * (remote_num_blocks or num_blocks)
),
transfers_per_block=2,
regions_per_block=1,
split_kv_regions=True,
)
if not is_mla_cache and len(shape) == 5 and shape[1] == 2:
num_blocks = shape[0]
if shape[2] == spec.block_size:
block_size, num_kv_heads, head_dim = shape[2:]
slot_size_bytes = num_kv_heads * head_dim * element_size
block_len = block_size * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[0],
local_kv_stride=stride[1],
remote_kv_stride=stride[1],
transfers_per_block=2,
regions_per_block=2,
split_kv_regions=False,
)
elif shape[3] == spec.block_size:
num_kv_heads, block_size, head_dim = shape[2:]
else:
kernel_num_blocks = num_blocks
kernel_block_size, _, _ = _select_kernel_block_layout(
layer_name, shape, spec
)
kernel_blocks_per_block = spec.block_size // kernel_block_size
if kernel_num_blocks % kernel_blocks_per_block != 0:
raise ValueError(
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
f"not divisible by {kernel_blocks_per_block}"
)
num_blocks = kernel_num_blocks // kernel_blocks_per_block
block_size = spec.block_size
block_stride = stride[0] * kernel_blocks_per_block
block_len = block_stride * element_size
slot_size_bytes = block_len // block_size
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=block_stride,
local_kv_stride=None,
remote_kv_stride=None,
transfers_per_block=1,
regions_per_block=1,
split_kv_regions=False,
)
slot_size_bytes = num_kv_heads * head_dim * element_size
block_len = block_size * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[0],
local_kv_stride=stride[1],
remote_kv_stride=stride[1],
transfers_per_block=2,
regions_per_block=2,
split_kv_regions=False,
)
if (
isinstance(spec, AttentionSpec)
and len(shape) == 4
and shape[1] == spec.num_heads
and shape[2] == spec.num_states
and shape[3] * element_size == spec.state_content_size_bytes
):
# Standardized per-layer [B, H, N, C] view (MLA is just H == 1).
num_blocks, num_heads, num_states, content_dim = shape
slot_size_bytes = num_heads * content_dim * element_size
block_len = num_states * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=spec.block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[0],
local_kv_stride=None,
remote_kv_stride=None,
transfers_per_block=1,
regions_per_block=1,
split_kv_regions=False,
)
cache_kind = "MLA" if is_mla_cache else "K/V"
raise ValueError(
f"Unsupported MoRIIO {cache_kind} cache shape for layer "
f"{layer_name}: {tuple(shape)}"
)
def kda_conv_ssm(
kv_cache: torch.Tensor,
spec: MambaSpec,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return zero-copy conv and SSM views over a packed two-state page."""
from math import prod
from vllm.utils.torch_utils import get_dtype_size
if len(spec.shapes) != 2 or len(spec.dtypes) != 2:
raise ValueError(
"MoRIIO KDA transfer requires exactly two states (conv and SSM)"
)
if kv_cache.element_size() != 1:
raise ValueError("MoRIIO KDA packed cache must use a byte-sized dtype")
if kv_cache.ndim != 4 or kv_cache.shape[1:3] != (1, 1) or kv_cache.stride(3) != 1:
raise ValueError(
"MoRIIO KDA packed cache must have shape [B, 1, 1, C] with "
"a contiguous byte dimension"
)
# Mirror MambaBase.bind_kv_cache: blocks may be strided within a shared
# multi-layer allocation, while every individual byte page is contiguous.
pages = kv_cache.squeeze(dim=(1, 2))
states: list[torch.Tensor] = []
offset = 0
for shp, dt in zip(spec.shapes, spec.dtypes, strict=True):
nbytes = prod(shp) * get_dtype_size(dt)
if offset + nbytes > pages.shape[1]:
raise ValueError(
"MoRIIO KDA state layout exceeds the packed cache page size"
)
state = pages[:, offset : offset + nbytes].view(dt)
states.append(state.view(-1, *shp))
offset += nbytes
return states[0], states[1]
def iter_layer_registration_regions(
layer_name: str,
kv_cache: torch.Tensor,
layer_to_spec: Mapping[str, KVCacheSpec],
) -> list[tuple[torch.Tensor, int]]:
spec = layer_to_spec[layer_name]
if isinstance(spec, MambaSpec):
# KDA layer: register the conv and ssm tensors as two whole-tensor
# regions (the conv 3-subprojection split is expressed as transfer
# offset triples, not extra registrations).
conv, ssm = kda_conv_ssm(kv_cache, spec)
geom = get_mamba_transfer_geometry(conv, ssm)
return [(conv, geom.conv_region_len), (ssm, geom.ssm_region_len)]
geometry = cast(
LayerTransferGeometry,
get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec),
)
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
if geometry.regions_per_block == 1:
# Padded/interleaved pages stride past block_len: register the strided
# span, ending at the last block's bytes so it stays in-allocation.
block_stride_bytes = geometry.block_stride * kv_cache.element_size()
region_len = max(
region_len,
(geometry.num_blocks - 1) * block_stride_bytes + geometry.block_len,
)
if geometry.split_kv_regions:
return [(cache, region_len) for cache in kv_cache]
return [(kv_cache, region_len)]
def merge_contiguous_offsets(
offsets_local: list[int],
offsets_remote: list[int],
sizes: list[int],
) -> tuple[list[int], list[int], list[int]]:
if not offsets_local:
return [], [], []
if not (len(offsets_local) == len(offsets_remote) == len(sizes)):
raise ValueError("Input list lengths mismatch")
rows = sorted(zip(offsets_local, offsets_remote, sizes), key=lambda row: row[0])
merged: list[list[int]] = []
for local, remote, size in rows:
if (
merged
and local == merged[-1][0] + merged[-1][2]
and remote == merged[-1][1] + merged[-1][2]
):
merged[-1][2] += size
else:
merged.append([local, remote, size])
return (
[row[0] for row in merged],
[row[1] for row in merged],
[row[2] for row in merged],
)
def compute_block_transfer_offsets(
layer_name: str,
kv_cache: torch.Tensor,
layer_to_spec: Mapping[str, KVCacheSpec],
local_block_ids: list[int],
remote_block_ids: list[int],
remote_num_blocks: int,
merge_fn: Callable[
[list[int], list[int], list[int]], tuple[list[int], list[int], list[int]]
] = merge_contiguous_offsets,
) -> tuple[list[int], list[int], list[int]]:
# A shorter/empty local list transfers only what decode allocated (READ-mode
# partial or full-prefix-hit); a longer local list is a genuine bug.
if len(local_block_ids) > len(remote_block_ids):
raise ValueError(
"local_block_ids longer than remote_block_ids: "
f"{len(local_block_ids)} > {len(remote_block_ids)}"
)
geometry = cast(
LayerTransferGeometry,
get_layer_transfer_geometry(
layer_name, kv_cache, layer_to_spec, remote_num_blocks
),
)
invalid_local = next(
(
block_id
for block_id in local_block_ids
if not 0 <= block_id < geometry.num_blocks
),
None,
)
if invalid_local is not None:
raise ValueError(
f"local block id {invalid_local} is outside [0, {geometry.num_blocks})"
)
invalid_remote = next(
(
block_id
for block_id in remote_block_ids
if not 0 <= block_id < remote_num_blocks
),
None,
)
if invalid_remote is not None:
raise ValueError(
f"remote block id {invalid_remote} is outside [0, {remote_num_blocks})"
)
element_size = kv_cache.element_size()
transfer_size_byte = geometry.block_len
per_block = geometry.transfers_per_block
total = len(local_block_ids) * per_block
offset_local = [0] * total
offset_remote = [0] * total
sizes = [transfer_size_byte] * total
w = 0
for lb, rb in zip(local_block_ids, remote_block_ids):
offset_local[w] = element_size * (lb * geometry.block_stride)
offset_remote[w] = element_size * (rb * geometry.block_stride)
w += 1
if per_block == 2:
assert geometry.local_kv_stride is not None
assert geometry.remote_kv_stride is not None
offset_local[w] = element_size * (
geometry.local_kv_stride + lb * geometry.block_stride
)
offset_remote[w] = element_size * (
geometry.remote_kv_stride + rb * geometry.block_stride
)
w += 1
return merge_fn(offset_local, offset_remote, sizes)
class MambaOffsetTemplate(NamedTuple):
"""Slot-independent conv+ssm offset decomposition for a KDA layer.
Homogeneous-TP geometry (conv/ssm slot strides, conv sub-projection
offsets, ssm per-slot size) is identical across requests and across the
homogeneous GDN layers, so it is computed once and reused; only the
per-request slot bases vary (see ``apply_mamba_offset_template``).
"""
num_slots: int
conv_slot_stride_bytes: int
ssm_slot_stride_bytes: int
ssm_read_bytes: int
conv_subprojs: tuple[tuple[int, int, int], ...]
def build_mamba_offset_template(
conv: torch.Tensor,
ssm: torch.Tensor,
split_info: MambaConvSplitInfo,
tp_ratio: int,
) -> MambaOffsetTemplate:
"""Precompute the slot-independent conv+ssm offset decomposition.
Only homogeneous TP (``tp_ratio == 1``) is supported; heterogeneous TP is
gated with ``NotImplementedError`` (see the design doc). The returned
template is a pure function of the layer geometry and ``split_info``, so it
can be cached and reused across requests without recomputation.
"""
if tp_ratio != 1:
# Heterogeneous-TP recurrent-state transfer needs the remote page's
# slot stride (which differs from the local page) and, for
# P_TP > D_TP, a multi-rank gather of conv/ssm slices. Both are
# follow-ups. The mamba path bypasses the attention KV-head validator,
# so fail loudly here instead of silently transferring wrong bytes.
raise NotImplementedError(
"heterogeneous-TP mamba/KDA transfer is not supported yet "
f"(tp_ratio={tp_ratio}); use equal prefill/decode TP for "
"hybrid models"
)
geom = get_mamba_transfer_geometry(conv, ssm)
conv_local_offsets = split_info.local_conv_offsets
conv_remote_offsets = split_info.remote_conv_offsets(0, 1)
ssm_read_bytes = geom.ssm_slot_bytes
conv_subprojs = tuple(
(loff, roff, rsz)
for (loff, _lsz), (roff, rsz) in zip(conv_local_offsets, conv_remote_offsets)
)
return MambaOffsetTemplate(
num_slots=conv.shape[0],
conv_slot_stride_bytes=geom.conv_slot_stride * geom.conv_element_size,
ssm_slot_stride_bytes=geom.ssm_slot_stride * geom.ssm_element_size,
ssm_read_bytes=ssm_read_bytes,
conv_subprojs=conv_subprojs,
)
def apply_mamba_offset_template(
template: MambaOffsetTemplate,
local_slots: list[int],
remote_slots: list[int],
) -> tuple[list[int], list[int], list[int]]:
"""Apply per-request slot bases to a cached ``MambaOffsetTemplate``.
Conv sub-projection entries (all slots) come first, followed by one ssm
entry per slot.
"""
if len(local_slots) != len(remote_slots):
raise ValueError(
"local and remote mamba slot counts must match: "
f"{len(local_slots)} != {len(remote_slots)}"
)
invalid_local = next(
(slot for slot in local_slots if not 0 <= slot < template.num_slots), None
)
if invalid_local is not None:
raise ValueError(
f"local mamba slot {invalid_local} is outside [0, {template.num_slots})"
)
invalid_remote = next((slot for slot in remote_slots if slot < 0), None)
if invalid_remote is not None:
raise ValueError(f"remote mamba slot {invalid_remote} must be non-negative")
# The peer's KDA slot count is not part of MoRIIOAgentMetadata yet. Do not
# compare remote slot IDs against the local pool size: two otherwise
# compatible engines may reserve different numbers of slots.
local_offs: list[int] = []
remote_offs: list[int] = []
sizes: list[int] = []
# Conv sub-projections (region 0), all slots first.
conv_slot_stride_bytes = template.conv_slot_stride_bytes
for ls, rs in zip(local_slots, remote_slots):
lbase = ls * conv_slot_stride_bytes
rbase = rs * conv_slot_stride_bytes
for loff, roff, rsz in template.conv_subprojs:
local_offs.append(lbase + loff)
remote_offs.append(rbase + roff)
sizes.append(rsz)
# SSM temporal state (region 1), one entry per slot.
ssm_slot_stride_bytes = template.ssm_slot_stride_bytes
ssm_read_bytes = template.ssm_read_bytes
for ls, rs in zip(local_slots, remote_slots):
local_offs.append(ls * ssm_slot_stride_bytes)
remote_offs.append(rs * ssm_slot_stride_bytes)
sizes.append(ssm_read_bytes)
return local_offs, remote_offs, sizes
def compute_mamba_conv_split_count(
local_slots: list[int],
split_info: MambaConvSplitInfo,
) -> int:
"""Number of leading conv entries in a KDA layer transfer plan.
Entries ``[:count]`` are conv sub-projections (conv region/session);
entries ``[count:]`` are the per-slot ssm state (ssm region/session).
"""
return len(local_slots) * len(split_info.local_conv_offsets)
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NIXL KV-cache transfer connector (disaggregated prefill / decode)."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
NixlBaseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector import (
NixlBaseConnector,
NixlConnector,
NixlPullConnector,
NixlPushConnector,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlAgentMetadata,
NixlConnectorMetadata,
NixlHandshakePayload,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
NixlPullConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
NixlPushConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
NixlPushConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
NixlConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
NixlKVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
__all__ = [
"NixlAgentMetadata",
"NixlBaseConnector",
"NixlBaseConnectorScheduler",
"NixlBaseConnectorWorker",
"NixlConnector",
"NixlConnectorMetadata",
"NixlConnectorScheduler",
"NixlConnectorWorker",
"NixlHandshakePayload",
"NixlKVConnectorStats",
"NixlPullConnector",
"NixlPullConnectorScheduler",
"NixlPullConnectorWorker",
"NixlPushConnector",
"NixlPushConnectorScheduler",
"NixlPushConnectorWorker",
]
@@ -1,549 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Base scheduler-side logic for the NIXL connector."""
import threading
import time
from typing import TYPE_CHECKING, Any
import msgspec
import zmq
from vllm import envs
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
EngineId,
clip_ssm_state_blocks,
yield_req_data,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorHandshakeMetadata,
KVConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
GET_META_MSG,
HeartbeatInfo,
NixlConnectorMetadata,
NixlHandshakePayload,
ReqId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import zmq_ctx
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.network_utils import make_zmq_path
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
MambaSpec,
SlidingWindowSpec,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
class NixlBaseConnectorScheduler:
"""Base implementation of Scheduler side methods shared by pull and push."""
# Emitted in kv_transfer_params so an external router can distinguish a
# pull (READ) producer from a push (WRITE) one. Overridden by the push
# scheduler.
_TRANSFER_MODE: str = "pull"
def __init__(
self,
vllm_config: "VllmConfig",
engine_id: str,
kv_cache_config: "KVCacheConfig",
):
self.vllm_config = vllm_config
parallel_config = vllm_config.parallel_config
# TP1 PCP+DCP exposes its DCP shards as transfer ranks.
self.transfer_tp_size = max(
parallel_config.tensor_parallel_size,
parallel_config.decode_context_parallel_size,
)
self.block_size = vllm_config.cache_config.block_size
self.engine_id: EngineId = engine_id
self.kv_cache_config = kv_cache_config
self.side_channel_host = envs.VLLM_NIXL_SIDE_CHANNEL_HOST
self.side_channel_port = (
envs.VLLM_NIXL_SIDE_CHANNEL_PORT
+ vllm_config.parallel_config.data_parallel_index
)
assert vllm_config.kv_transfer_config is not None
self._kv_lease_duration: int = (
vllm_config.kv_transfer_config.get_from_extra_config(
"kv_lease_duration", 30
)
)
# NOTE (NickLucche): For now we use a hardcoded value for a simpler interface.
self._heartbeat_interval = self._kv_lease_duration // 6
if current_platform.device_type == "cpu":
self.use_host_buffer = False
else:
self.use_host_buffer = (
vllm_config.kv_transfer_config.kv_buffer_device == "cpu"
)
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
# Also handle unlikely SW-only model case instead of checking num_groups>1.
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.transfer_groups
)
)
self._has_mamba = kv_cache_config.has_mamba_layers
self._bounded_replay = any(
g.kv_cache_spec.prefix_replay_tokens > 0
for g in kv_cache_config.kv_cache_groups
)
logger.info("Initializing NIXL Scheduler %s", engine_id)
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
logger.info("Hybrid Memory Allocator is enabled with NIXL")
# Background thread for handling new handshake requests.
self._nixl_handshake_listener_t: threading.Thread | None = None
self._stop_event = threading.Event()
# Requests that need to start recv/send.
# New requests are added by update_state_after_alloc in
# the scheduler. Used to make metadata passed to Worker.
self._reqs_need_recv: dict[
ReqId, tuple[Request, BlockIds, tuple[int, ...], bool]
] = {}
self._reqs_need_save: dict[ReqId, Request] = {}
# Reqs to send and their expiration time
self._reqs_need_send: dict[ReqId, float] = {}
self._reqs_in_batch: set[ReqId] = set()
# Reqs to remove from processed set because they're not to send after
# remote prefill or aborted.
self._reqs_not_processed: set[ReqId] = set()
# Heartbeat tracking: requests needing periodic lease-renewal heartbeats to
# remote P-side, stored as ready-to-send HeartbeatInfo grouped by remote engine
self._heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
# Reverse lookup: local req_id -> (engine_id, remote_req_id) for O(1) removal
self._heartbeat_req_engine: dict[ReqId, tuple[EngineId, ReqId]] = {}
self._last_heartbeat_time: float = 0.0
# Gather Sliding Window sizes for each kv cache group (if any) in number of
# blocks per KV cache group. This is used to clip the local attention window.
sw_sizes_tokens: list[tuple[int, int]] = [
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
else (0, self.block_size)
for g in kv_cache_config.transfer_groups
]
# cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively
# account for boundary overlap eg window isn't fully aligned with blocks.
self.blocks_per_sw = [
cdiv(n_tokens, block_size) + 1 if n_tokens else 0
for n_tokens, block_size in sw_sizes_tokens
]
# Trailing scratch slots that mamba managers co-allocate per request
# for speculative decoding; None for non-SSM groups.
self._ssm_spec_blocks = [
g.kv_cache_spec.num_speculative_blocks
if isinstance(g.kv_cache_spec, MambaSpec)
else None
for g in kv_cache_config.transfer_groups
]
# Threshold to decide whether to compute kv cache locally
# or pull from a remote node: minimum number of remote
# tokens to amortize the xfer latencies
self.kv_recompute_threshold: int = int(
vllm_config.kv_transfer_config.get_from_extra_config(
"kv_recompute_threshold", 64
)
)
# Bi-directional KV transfer feature supports KV block
# transfers from D node to P node
self.is_bidirectional_kv_xfer_enabled = (
vllm_config.kv_transfer_config.get_from_extra_config(
"bidirectional_kv_xfer", False
)
)
self.decoder_kv_blocks_ttl = (
vllm_config.kv_transfer_config.get_from_extra_config(
"decoder_kv_blocks_ttl", 480
)
)
if self.is_bidirectional_kv_xfer_enabled and self.kv_recompute_threshold > 0:
logger.info(
"Bidirectional KV transfer is enabled and the kv "
"recompute threshold is set to %d tokens."
"KV blocks on D are released after a TTL of %d seconds.",
self.kv_recompute_threshold,
self.decoder_kv_blocks_ttl,
)
def shutdown(self):
self._stop_event.set()
if self._nixl_handshake_listener_t is not None:
self._nixl_handshake_listener_t.join()
self._nixl_handshake_listener_t = None
def on_new_request(self, request: "Request") -> None:
"""Track a request that may need heartbeats."""
params = request.kv_transfer_params
if params is not None and params.get("do_remote_decode"):
self._truncate_request_for_prefill(request)
# NOTE (NickLucche) This excludes request meant for P, ie heartbeats are
# effectively disabled for Bidirectional KV transfer.
if params is None or not params.get("do_remote_prefill"):
return
# Only track if all required remote fields are present.
remote_engine_id = params.get("remote_engine_id")
remote_request_id = params.get("remote_request_id")
host = params.get("remote_host")
port = params.get("remote_port")
tp_size = params.get("tp_size")
dcp_size = params.get("dcp_size", 1)
pp_size = params.get("pp_size", 1)
if (
remote_engine_id is None
or remote_request_id is None
or host is None
or port is None
or tp_size is None
):
return
if remote_engine_id not in self._heartbeat_by_engine:
self._heartbeat_by_engine[remote_engine_id] = HeartbeatInfo(
req_ids=set(),
host=host,
port=port,
tp_size=tp_size,
dcp_size=dcp_size,
pp_size=pp_size,
)
self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id)
self._heartbeat_req_engine[request.request_id] = (
remote_engine_id,
remote_request_id,
)
def _stop_heartbeat(self, req_id: ReqId) -> None:
"""Remove *req_id* from heartbeat tracking (if tracked)."""
if key := self._heartbeat_req_engine.pop(req_id, None):
engine_id, remote_id = key
if info := self._heartbeat_by_engine.get(engine_id):
info.req_ids.discard(remote_id)
if not info.req_ids:
# Clean up empty engines so we don't leak a key when remote dies.
del self._heartbeat_by_engine[engine_id]
def get_exchange_clipped_blocks(
self, block_ids: BlockIds, clip_ssm: bool = True
) -> BlockIds:
"""Clip a request's block lists down to the transferable blocks.
Sliding-window groups keep only the in-window tail: the KV cache
manager allocates blocks for the entire sequence length and cleans up
out-of-window blocks only prior to the `request_finished_all_groups`
hook.
SSM groups keep only their state-bearing slot: the trailing
speculative scratch slots and everything before the running state
(null placeholders and the previous step's superseded state) go.
Use this at every block-id exchange point. Pass ``clip_ssm=False``
for per-step partial lists (host-buffer save), where the SSM strip
does not apply.
"""
if len(block_ids) == 0:
# No blocks to clip, e.g. a full prefix cache hit.
return block_ids
block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
if not self._is_hma_required:
return block_ids
# NOTE (NickLucche) This logic is currently handled at the connector level
# because offloading connectors might want to receive the whole sequence even
# for SWA groups. We will abstract this logic once the interface is more stable
assert len(block_ids) == len(self.blocks_per_sw), (
"Number of KV cache groups must match"
)
clipped = []
for i, blocks in enumerate(block_ids):
if n_sw := self.blocks_per_sw[i]:
blocks = blocks[-n_sw:]
elif (
clip_ssm
and blocks
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
):
blocks = clip_ssm_state_blocks(blocks, n_spec_blocks)
clipped.append(blocks)
return tuple(clipped)
def set_xfer_handshake_metadata(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""Set the KV connector handshake metadata for this connector.
Args:
metadata (dict): the handshake metadata to set.
"""
encoded_data: dict[tuple[int, int], bytes] = {}
encoder = msgspec.msgpack.Encoder()
for (pp_rank, tp_rank), rank_metadata in metadata.items():
if not isinstance(rank_metadata, NixlHandshakePayload):
raise ValueError(
"NixlConnectorScheduler expects NixlHandshakePayload for "
"handshake metadata."
)
encoded_data[(pp_rank, tp_rank)] = encoder.encode(rank_metadata)
logger.debug(
"PP rank %d, TP rank %d: encoded NixlHandshakePayload size: %s bytes",
pp_rank,
tp_rank,
str(len(encoded_data[(pp_rank, tp_rank)])),
)
# Only start the listener when we have metadata to serve.
if self._nixl_handshake_listener_t is None:
ready_event = threading.Event()
self._nixl_handshake_listener_t = threading.Thread(
target=self._nixl_handshake_listener,
args=(
encoded_data,
ready_event,
self._stop_event,
self.side_channel_host,
self.side_channel_port,
),
daemon=True,
name="nixl_handshake_listener",
)
self._nixl_handshake_listener_t.start()
ready_event.wait() # Wait for listener ZMQ socket to be ready.
@staticmethod
def _nixl_handshake_listener(
encoded_data: dict[tuple[int, int], Any],
ready_event: threading.Event,
stop_event: threading.Event,
host: str,
port: int,
):
"""Background thread for getting new NIXL handshakes."""
# NOTE(rob): this is a simple implementation. We will move
# to a better approach via HTTP endpoint soon.
# Listen for new requests for metadata.
path = make_zmq_path("tcp", host, port)
logger.debug("Starting listening on path: %s", path)
with zmq_ctx(zmq.ROUTER, path) as sock:
sock.setsockopt(zmq.RCVTIMEO, 1000)
ready_event.set()
while True:
try:
identity, _, msg = sock.recv_multipart()
except zmq.Again:
if stop_event.is_set():
break
continue
# Decode (GET_META_MSG, pp_rank, tp_rank).
msg, target_pp_rank, target_tp_rank = msgspec.msgpack.decode(msg)
logger.debug(
"Received message for pp rank %s, tp rank %s",
target_pp_rank,
target_tp_rank,
)
if msg != GET_META_MSG:
logger.warning("Connection listener got unexpected message %s", msg)
# Echo our perf_counter so P can estimate the clock offset.
# perf_counter is only comparable within a process, so this
# listener must run in the same process that stamps the block
# expiry deadline (`_reqs_need_send`).
ts = msgspec.msgpack.encode(time.perf_counter())
sock.send_multipart(
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)], ts)
)
def _prefill_backoff(self) -> int:
"""Trailing prompt tokens the prefiller must not compute; the decoder
recomputes them locally.
Mamba needs h(N-1) so the decoder can derive h(N) itself. DSV41's sliding
window under bounded replay is not replayed after a P/D load, so it must
be laid out for the token the decoder recomputes. Multi-module
MTP needs to keep its whole lookahead window off of the prefiller, which
would otherwise embed the unverified drafts in the MTP layer's KV cache. The
decoder would never rebuild them, because the update is sized by the rejection
count, which is zero for the first decode.
"""
return max(
1 if self._has_mamba or self._bounded_replay else 0,
self.vllm_config.num_prefill_lookahead_tokens - 1,
)
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. The number of prompt tokens to load from the prefiller.
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
will recompute locally."""
backoff = self._prefill_backoff()
if backoff and num_prompt_tokens > backoff:
return num_prompt_tokens - backoff
return num_prompt_tokens
def _truncate_request_for_prefill(self, request: "Request") -> None:
"""P-side only: drop the trailing ``_prefill_backoff()`` prompt tokens
so the prefiller stops short of what the decoder recomputes locally.
For Mamba that is the single token needed to yield h(N-1); for
multi-module MTP it is the drafter's whole lookahead window.
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
request is preempted and rescheduled."""
backoff = self._prefill_backoff()
params = request.kv_transfer_params
if (
backoff
and params is not None
# Guard against repeated truncation after preemption/reschedule.
and not params.get("_p_side_truncated")
and request.num_prompt_tokens > backoff
# A request that skips reading the prefix cache (prompt logprobs)
# loads nothing on the decoder, which recomputes its whole prompt.
and not request.get_skip_reading_prefix_cache()
):
if request.prompt_token_ids is not None:
del request.prompt_token_ids[-backoff:]
elif request.prompt_embeds is not None:
request.prompt_embeds = request.prompt_embeds[:-backoff]
else:
return
del request._all_token_ids[-backoff:]
request.num_prompt_tokens -= backoff
request.max_tokens = 1
params["_p_side_truncated"] = True
def _build_save_meta(
self,
meta: NixlConnectorMetadata,
scheduler_output: SchedulerOutput,
) -> None:
# only called when use_host_buffer is True to build the save metadata
# NOTE: For the prefill side, there might be a chance that an early added
# request is a chunked prefill, so we need to check if new blocks are added
for req_id, new_block_id_groups, _ in yield_req_data(scheduler_output):
req_to_save = self._reqs_need_save.get(req_id)
if req_to_save is None or new_block_id_groups is None:
continue
req = req_to_save
assert req.kv_transfer_params is not None
clipped_block_id_groups = self.get_exchange_clipped_blocks(
new_block_id_groups, clip_ssm=False
)
meta.add_new_req_to_save(
request_id=req_id,
local_block_ids=clipped_block_id_groups,
kv_transfer_params=req.kv_transfer_params,
)
assert scheduler_output.num_scheduled_tokens is not None
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
is_partial = (
req.num_computed_tokens + num_scheduled_tokens
) < req.num_prompt_tokens
if not is_partial:
# For non-partial prefills, once new req_meta is scheduled, it
# can be removed from _reqs_need_save.
# For partial prefill case, we will retain the request in
# _reqs_need_save until all blocks are scheduled with req_meta.
# Therefore, only pop if `not is_partial`.
self._reqs_need_save.pop(req_id)
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
meta = NixlConnectorMetadata()
# Loop through scheduled reqs and convert to ReqMeta.
for req_id, (
req,
block_ids,
cached,
awaiting_kvs,
) in self._reqs_need_recv.items():
assert req.kv_transfer_params is not None
meta.add_new_req_to_recv(
request_id=req_id,
local_block_ids=block_ids,
kv_transfer_params=req.kv_transfer_params,
local_num_computed_blocks=cached,
awaiting_kvs=awaiting_kvs,
)
if self.use_host_buffer:
self._build_save_meta(meta, scheduler_output)
meta.reqs_to_send = self._reqs_need_send
# Clock reference for reqs_to_send: deadlines above are in this
# process's perf_counter domain; workers (possibly on other nodes,
# where perf_counter has a different epoch) rebase against this.
meta.scheduler_clock = time.perf_counter()
meta.reqs_in_batch = self._reqs_in_batch
meta.reqs_not_processed = self._reqs_not_processed
# Package heartbeats, throttled by heartbeat_interval.
if self._heartbeat_by_engine:
now = time.perf_counter()
if now - self._last_heartbeat_time >= self._heartbeat_interval:
self._last_heartbeat_time = now
meta.heartbeat_by_engine = self._heartbeat_by_engine
# Clear the list once workers start the transfers
self._reqs_need_recv.clear()
self._reqs_in_batch = set()
self._reqs_not_processed = set()
self._reqs_need_send = {}
return meta
def update_connector_output(self, connector_output: "KVConnectorOutput") -> None:
"""Stop heartbeating for requests whose KV transfer completed."""
for req_id in connector_output.finished_recving or ():
self._stop_heartbeat(req_id)
def has_pending_push_work(self) -> bool:
return False
############################################################
# Abstract methods that subclasses must implement
############################################################
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int, bool]:
raise NotImplementedError
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
raise NotImplementedError
def request_finished(
self,
request: "Request",
block_ids: BlockIds,
) -> tuple[bool, dict[str, Any] | None]:
raise NotImplementedError
File diff suppressed because it is too large Load Diff
@@ -1,426 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NIXL connector facades.
This module hosts the thin facade classes that vLLM's KV-connector layer
instantiates. Almost all the real work lives in the per-mode scheduler
and worker classes; the connector classes here only forward calls.
* :class:`NixlBaseConnector` – common logic shared by pull and push.
* :class:`NixlPullConnector` – pull-based (READ) KV transfer.
* :class:`NixlPushConnector` – push-based (WRITE) KV transfer.
* ``NixlConnector`` – backward-compatible alias for :class:`NixlPullConnector`.
"""
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
CopyBlocksOp,
KVConnectorBase_V1,
KVConnectorHandshakeMetadata,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
NixlPullConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
NixlPushConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
NixlPushConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
NixlKVConnectorStats,
NixlPromMetrics,
)
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
NixlBaseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
"""Base connector with common logic shared by pull and push modes."""
_cache_hit_source = CacheHitSource.P2P
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
return True
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, role, kv_cache_config)
assert vllm_config.kv_transfer_config is not None
assert vllm_config.kv_transfer_config.engine_id is not None
if vllm_config.kv_transfer_config.kv_role == "kv_both":
logger.warning_once(
"Using kv_role='kv_both' with NixlConnector is deprecated "
"and will be removed in a future release. Please set "
"kv_role='kv_producer' for prefill instances and "
"kv_role='kv_consumer' for decode instances. "
)
parallel_config = vllm_config.parallel_config
pcp_size = parallel_config.prefill_context_parallel_size
kv_role = vllm_config.kv_transfer_config.kv_role
if pcp_size > 1 and kv_role in ("kv_consumer", "kv_both"):
raise NotImplementedError(
"NixlConnector PCP currently supports kv_producer only. "
"Consumers and kv_both require "
"prefill_context_parallel_size=1."
)
dcp_size = parallel_config.decode_context_parallel_size
if (
pcp_size > 1
and dcp_size > 1
and (parallel_config.tensor_parallel_size != 1 or dcp_size != pcp_size)
):
raise NotImplementedError(
"NixlConnector PCP+DCP requires TP1 with DCP spanning "
"the full PCP group."
)
# TODO: Support PCP with bidirectional KV transfer by tracking separate
# send and receive completion counts.
if pcp_size > 1 and vllm_config.kv_transfer_config.get_from_extra_config(
"bidirectional_kv_xfer", False
):
raise NotImplementedError(
"NixlConnector PCP producers do not support bidirectional KV transfer."
)
self.kv_cache_config = kv_cache_config
self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id
self.kv_transfer_config = vllm_config.kv_transfer_config
# Subclasses must set self.connector_scheduler and self.connector_worker
self.connector_scheduler: NixlBaseConnectorScheduler | None = None
self.connector_worker: NixlBaseConnectorWorker | None = None
############################################################
# Class Methods
############################################################
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
return False
@classmethod
def get_required_kvcache_layout(cls, vllm_config: VllmConfig):
if vllm_config.model_config is None:
logger.warning_once(
"Unable to detect current VLLM config. "
"Fallback to default kv cache layout."
)
return None
use_mla = vllm_config.model_config.use_mla
if use_mla:
# return None when we have mla
# as the layout should not matter in that case,
# which fallback to the default behavior.
return None
logger.info_once(
"NixlConnector setting KV cache layout to LBHNC for "
"better xfer performance."
)
return "LBHNC"
############################################################
# Scheduler Side Methods
############################################################
def get_finished_count(self) -> int | None:
return None
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
return self._kv_cache_config.transfer_group_ids
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
assert self.connector_scheduler is not None
return self.connector_scheduler.update_state_after_alloc(
request, blocks, num_external_tokens
)
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def on_new_request(self, request: "Request") -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.on_new_request(request)
def update_connector_output(self, connector_output: KVConnectorOutput):
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, (block_ids,))
def request_finished_all_groups(
self,
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
def set_xfer_handshake_metadata_pp_aware(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""Set handshake metadata keyed by (pp_rank, tp_rank) so the side
channel can serve every PP stage's agent metadata.
Args:
metadata (dict): the handshake metadata to set.
"""
assert self.connector_scheduler is not None
self.connector_scheduler.set_xfer_handshake_metadata(metadata)
############################################################
# Worker Side Methods
############################################################
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
assert self.connector_worker is not None
self.connector_worker.set_host_xfer_buffer_ops(copy_operation)
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
"""Get the finished recving and sending requests."""
assert self.connector_worker is not None
return self.connector_worker.get_finished()
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
results = self.connector_worker.get_transfer_results()
if (
self.kv_transfer_config.kv_role == "kv_producer"
and self.connector_worker.pcp_rank > 0
and not self.connector_worker.pcp_dcp_sharded
):
results.finished_sending.clear()
return results
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get block IDs that failed to load via NIXL."""
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
NixlKVConnectorStats(data=data)
if data is not None
else NixlKVConnectorStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return NixlPromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def wait_for_layer_load(self, layer_name: str) -> None:
"""NixlConnector does not do layerwise saving."""
pass
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs,
) -> None:
"""NixlConnector does not save explicitly."""
pass
def wait_for_save(self):
assert self.connector_worker is not None
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
if self.connector_worker.use_host_buffer and self.connector_worker.copy_blocks:
self.connector_worker.save_kv_to_host(self._connector_metadata)
def has_pending_push_work(self) -> bool:
if self.connector_scheduler is not None:
return self.connector_scheduler.has_pending_push_work()
return False
def shutdown(self):
if self.connector_worker is not None:
self.connector_worker.shutdown()
if self.connector_scheduler is not None:
self.connector_scheduler.shutdown()
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
"""Get the KVConnector handshake metadata for this connector.
This metadata is used for out-of-band connector handshake
between P/D workers.
Returns:
KVConnectorHandshakeMetadata: the handshake metadata.
None if no handshake metadata is available.
"""
assert self.connector_worker is not None
if (
self.kv_transfer_config.kv_role == "kv_producer"
and self.connector_worker.pcp_rank > 0
and not self.connector_worker.pcp_dcp_sharded
):
return None
return self.connector_worker.xfer_handshake_metadata
class NixlPullConnector(NixlBaseConnector):
"""Pull-based (READ) NIXL KV transfer connector."""
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, role, kv_cache_config)
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = NixlPullConnectorScheduler(
vllm_config, self.engine_id, kv_cache_config
)
self.connector_worker = None
elif role == KVConnectorRole.WORKER:
self.connector_scheduler = None
self.connector_worker = NixlPullConnectorWorker(
vllm_config, self.engine_id, kv_cache_config
)
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
assert self.connector_worker is not None
assert isinstance(self.connector_worker, NixlPullConnectorWorker)
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
self.connector_worker.start_load_kv(self._connector_metadata)
class NixlPushConnector(NixlBaseConnector):
"""Push-based (WRITE) NIXL KV transfer connector."""
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, role, kv_cache_config)
if vllm_config.parallel_config.decode_context_parallel_size > 1:
raise ValueError(
"NixlPushConnector does not support decode_context_parallel_size > 1."
)
self.connector_scheduler: NixlPushConnectorScheduler | None = None
self.connector_worker: NixlPushConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = NixlPushConnectorScheduler(
vllm_config, self.engine_id, kv_cache_config
)
elif role == KVConnectorRole.WORKER:
self.connector_worker = NixlPushConnectorWorker(
vllm_config, self.engine_id, kv_cache_config
)
else:
raise ValueError(f"Unsupported KVConnectorRole: {role}")
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
"""Drive push processing on the worker.
The worker enqueues registrations / finished blocks for the
background ``nixl-push-writer`` thread; the writer issues the
WRITE transfers and polls NIXL notifs without further
engine-thread involvement.
"""
assert self.connector_worker is not None
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
self.connector_worker.start_load_kv(self._connector_metadata)
# Backward compatibility: NixlConnector is the pull-based connector.
NixlConnector = NixlPullConnector
__all__ = [
"NixlBaseConnector",
"NixlConnector",
"NixlPullConnector",
"NixlPushConnector",
]
@@ -1,340 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Metadata dataclasses and helpers for the NIXL connector."""
from dataclasses import dataclass, field
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds, EngineId
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorHandshakeMetadata,
KVConnectorMetadata,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
TransferHandle = int
ReqId = str
GET_META_MSG = b"get_meta_msg"
# Push-mode (WRITE-based) registration notification.
# Sent worker-to-worker over NIXL: D worker -> P worker, encoded as
# PUSH_REG_NOTIF_PREFIX + msgpack(registration_data).
PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
#
# NIXL Connector Version
#
# Increment this version whenever there is an incompatible change to:
# - NixlAgentMetadata schema
# - kv_transfer_params schema or semantics
# - NIXL transfer protocol or wire format
# - KV cache memory layout or block organization
# - Any other change that breaks P/D interoperability
#
# Version History:
# 1: Initial version with compatibility checking
# 2: Add remote_request_id to kv_transfer_params
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
# 4: Add KV block lease renewal through heartbeats
# 5: Add remote_blocks_expiry_time to kv_transfer_params + handshake
# clock-sync timestamp
# 6: Validate EAGLE/MTP speculative configuration compatibility
# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash
# 8: Add dcp_size and pcp_size to NixlAgentMetadata
# 9: Add block_strides
# 10: Add dense virtual transfer pages for compressed MLA caches
# 11: Add per-region transfer geometry and memory types to NixlAgentMetadata
# 12: Add per-region member names for PP push
# 13: Add packed-member layouts and order-independent packed-push backend hashes
# 14: Add Qwen3.8-Flash-Next PLE page length - #59997
#
NIXL_CONNECTOR_VERSION: int = 14
@dataclass
class NixlAgentMetadata:
engine_id: str
agent_metadata: bytes
kv_caches_base_addr: list[int]
device_id: int
num_blocks: int
block_lens: list[int]
block_strides: list[int]
kv_cache_layout: str
block_size: int
ssm_sizes: tuple[int, int]
attn_backend_name: str
physical_blocks_per_logical_kv_block: int
region_num_blocks: list[int] | None = None
region_group_ids: list[int] | None = None
region_names: list[str] | None = None
region_mem_types: list[str] | None = None
dcp_size: int = 1
pcp_size: int = 1
# Page length of the PLE short-conv state (None when the model has none).
ple_block_len: int | None = None
# Layer names sharing each advertised region, in region order.
region_members: list[list[str]] = field(default_factory=list)
# Packed member -> (byte offset in its region's block, bytes per page).
packed_member_layouts: dict[str, tuple[int, int]] = field(default_factory=dict)
@dataclass
class NixlHandshakePayload(KVConnectorHandshakeMetadata):
"""Wrapper for NIXL handshake sent over the wire.
Enables two-phase decoding for graceful compatibility checking:
1. Decode NixlHandshakePayload to get compatibility_hash
2. Compute local hash and compare
3. Only if hashes match, decode agent_metadata_bytes
This prevents decoder errors when NixlAgentMetadata schema is
incompatible, allowing graceful failure with clear error message.
"""
compatibility_hash: str
agent_metadata_bytes: bytes # NixlAgentMetadata encoded
def _get_speculative_compatibility_factors(
vllm_config: VllmConfig,
) -> dict[str, Any] | None:
"""Return NIXL compatibility factors for hidden-state-based speculators."""
speculative_config = vllm_config.speculative_config
if speculative_config is None or not speculative_config.use_eagle():
return None
draft_model_config = speculative_config.draft_model_config
assert draft_model_config is not None
auxiliary_layer_ids = getattr(
draft_model_config.hf_config,
"eagle_aux_hidden_state_layer_ids",
None,
)
# kv_cache_dtype is a user override that defaults to None, meaning "inherit
# the target's --kv-cache-dtype". Resolve it to the effective value so an
# explicit setting on one side and inheritance on the other (same effective
# dtype) don't spuriously mismatch.
kv_cache_dtype = (
speculative_config.kv_cache_dtype or vllm_config.cache_config.cache_dtype
)
# Note: the draft attention_backend is intentionally not hashed. Its only
# transfer-relevant effect is the KV block layout/size, which is validated
# per region at runtime in _validate_remote_agent_handshake. The connector
# only sees the raw override here (usually None = auto-select), never the
# resolved backend, so hashing it would cause false mismatches without
# catching anything the runtime layout check misses.
return {
"method": speculative_config.method,
"model": draft_model_config.model,
"revision": draft_model_config.revision,
"code_revision": draft_model_config.code_revision,
"parallel_drafting": speculative_config.parallel_drafting,
"kv_cache_dtype": str(kv_cache_dtype),
"auxiliary_layer_ids": (
tuple(auxiliary_layer_ids) if auxiliary_layer_ids is not None else None
),
}
def compute_nixl_compatibility_hash(
vllm_config: VllmConfig,
attn_backend_name: str,
transfer_mode: str = "pull",
) -> str:
"""Compute compatibility hash for NIXL KV transfer.
Hash only the factors that affect whether two NIXL instances can
successfully transfer KV cache data.
Factors included:
- vLLM version and NIXL connector version
- Model architecture (name, dtype, KV heads, layers)
- KV cache format (dtype, sliding window)
- Attention backend
- EAGLE/MTP configuration that affects transferred state
- Transfer mode (push vs pull)
The transfer mode is included because the push (WRITE) and pull (READ)
connectors use incompatible transfer protocols; a push connector and a
pull connector must never complete a handshake with each other.
Note: Factors like tensor_parallel_size, block_size, and kv_cache_layout
are validated at runtime in _validate_remote_agent_handshake and are not
included in this hash to support heterogeneous deployments.
Note - the set of factors are likely to evolve significantly over
time to be more or less permissive.
Returns:
SHA-256 hex digest
"""
from vllm import __version__ as vllm_version
from vllm.config.utils import hash_factors
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
is_hma_enabled = not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
factors = {
# Version compatibility
"vllm_version": vllm_version,
"nixl_connector_version": NIXL_CONNECTOR_VERSION,
# Model architecture - affects KV cache shape
"model": model_config.model,
"dtype": str(model_config.dtype),
"num_kv_heads": model_config.get_total_num_kv_heads(),
"head_size": model_config.get_head_size(),
"num_hidden_layers": model_config.get_total_num_hidden_layers(),
# Attention backend and KV cache dtype affect memory layout
"attn_backend_name": attn_backend_name,
"cache_dtype": str(cache_config.cache_dtype),
"is_hma_enabled": is_hma_enabled,
"speculative_config": _get_speculative_compatibility_factors(vllm_config),
# push (WRITE) and pull (READ) connectors are protocol-incompatible
"transfer_mode": transfer_mode,
}
compat_hash = hash_factors(factors)
logger.debug(
"NIXL compatibility hash: %s (model=%s, dtype=%s, num_kv_heads=%d, "
"cache_dtype=%s, attn_backend=%s)",
compat_hash,
factors["model"],
factors["dtype"],
factors["num_kv_heads"],
factors["cache_dtype"],
attn_backend_name,
)
return compat_hash
@dataclass
class HeartbeatInfo:
"""Heartbeat data for a single remote engine, sent from D worker to P."""
req_ids: set[ReqId]
host: str
port: int
tp_size: int
dcp_size: int = 1
pp_size: int = 1
@dataclass
class RemoteMeta:
block_ids: BlockIds
host: str
port: int
engine_id: str
request_id: str
blocks_expiry_time: float | None = None
num_tokens: int | None = None
@dataclass
class ReqMeta:
local_block_ids: BlockIds
# To be used when logical block size does not match the kernel block size
local_physical_block_ids: BlockIds
tp_size: int
dcp_size: int = 1
# Per-KV-cache-group logical blocks this rank already holds, i.e. its
# prefix-cache hit. Fixes where this rank's DCP slice starts relative to
# the remote's; kept per-group since hybrid models (e.g. SWA+FA) can have
# different cache-hit counts per group.
local_num_computed_blocks: tuple[int, ...] = ()
remote: RemoteMeta | None = None
# Remote block size, discovered during NIXL handshake (push mode).
remote_block_size: int | None = None
# Remote producer pipeline-parallel size (push mode, D side).
pp_size: int = 1
# True only when the scheduler parked the request in WAITING_FOR_REMOTE_KVS
# and expects it in finished_recving; notify-only recvs must not be reported.
awaiting_kvs: bool = False
# Worker-only, per-region physical pages to zero after a successful pull.
# None selects group-based completion; empty lists mean no zeroing.
region_blocks_to_zero: BlockIds | None = None
class NixlConnectorMetadata(KVConnectorMetadata):
def __init__(self):
self.reqs_to_recv: dict[ReqId, ReqMeta] = {}
self.reqs_to_save: dict[ReqId, ReqMeta] = {}
self.reqs_to_send: dict[ReqId, float] = {}
# The scheduler process's time.perf_counter() when this metadata was
# built. reqs_to_send deadlines are stamped with the scheduler's
# clock, which is NOT comparable across processes (perf_counter is
# process/boot-local): workers must rebase the remaining TTL onto
# their own clock via this reference. 0.0 = unset (legacy metadata).
self.scheduler_clock: float = 0.0
self.reqs_in_batch: set[ReqId] = set()
self.reqs_not_processed: set[ReqId] = set()
# Heartbeat data grouped by remote engine, sent by D worker to P.
self.heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
# Push mode (D side): registration data the D worker should send to
# P workers via NIXL notification on this step.
self.push_registrations: dict[ReqId, dict[str, Any]] = {}
# Push mode (P side): newly finished request blocks to be matched
# against pending D registrations on the P worker.
self.push_finished_blocks: dict[ReqId, BlockIds] = {}
def _add_new_req(
self,
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
local_num_computed_blocks: tuple[int, ...] = (),
awaiting_kvs: bool = False,
) -> ReqMeta:
return ReqMeta(
local_block_ids=local_block_ids,
local_physical_block_ids=local_block_ids,
# P workers don't need to receive these from proxy here.
tp_size=kv_transfer_params.get("tp_size", 1),
dcp_size=kv_transfer_params.get("dcp_size", 1),
remote_block_size=kv_transfer_params.get("remote_block_size"),
pp_size=kv_transfer_params.get("pp_size", 1),
local_num_computed_blocks=local_num_computed_blocks,
awaiting_kvs=awaiting_kvs,
)
def add_new_req_to_save(
self,
request_id: ReqId,
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
):
self.reqs_to_save[request_id] = self._add_new_req(
local_block_ids, kv_transfer_params
)
def add_new_req_to_recv(
self,
request_id: ReqId,
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
local_num_computed_blocks: tuple[int, ...] = (),
awaiting_kvs: bool = False,
):
req = self._add_new_req(
local_block_ids,
kv_transfer_params,
local_num_computed_blocks,
awaiting_kvs,
)
req.remote = RemoteMeta(
block_ids=kv_transfer_params["remote_block_ids"],
engine_id=kv_transfer_params["remote_engine_id"],
request_id=kv_transfer_params["remote_request_id"],
host=kv_transfer_params["remote_host"],
port=kv_transfer_params["remote_port"],
blocks_expiry_time=kv_transfer_params.get("remote_blocks_expiry_time"),
num_tokens=kv_transfer_params.get("remote_num_tokens"),
)
self.reqs_to_recv[request_id] = req
@@ -1,292 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pull-specific scheduler-side logic for the NIXL connector."""
import time
from typing import TYPE_CHECKING, Any
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
NixlBaseConnectorScheduler,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
"""Pull-specific scheduler logic (READ-based KV transfer)."""
def __init__(
self,
vllm_config: "VllmConfig",
engine_id: str,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, engine_id, kv_cache_config)
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int, bool]:
"""For remote prefill, pull all prompt blocks from remote
asynchronously relative to engine execution.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally
computed tokens for this request
Returns:
* the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
* true if the external KV cache tokens will be loaded
asynchronously (between scheduler steps).
"""
params = request.kv_transfer_params
logger.debug(
"NIXLConnector get_num_new_matched_tokens: "
"num_computed_tokens=%s, kv_transfer_params=%s",
num_computed_tokens,
params,
)
if params is not None and params.get("do_remote_prefill"):
# Remote prefill: get all prompt blocks from remote.
token_ids = request.prompt_token_ids or []
actual = self._get_remote_prefill_token_count(len(token_ids))
count = actual - num_computed_tokens
if count > 0:
return count, True
if (
params is not None
and params.get("do_remote_decode")
and params.get("remote_block_ids")
and all(
p in params
for p in (
"remote_engine_id",
"remote_request_id",
"remote_host",
"remote_port",
)
)
):
# Decode node has kv blocks for part of prefill request, so, provide them
# as an external token count to scheduler.
# The tokens will be loaded if not already present
# in the prefill node local cache
remote_num_tokens = params.get("remote_num_tokens") or 0
count = (
min(remote_num_tokens, request.num_prompt_tokens) - num_computed_tokens
)
if count > 0:
# Check kv_recompute_threshold: skip pull if
# remote tokens are below the threshold.
if (
self.kv_recompute_threshold > 0
and count < self.kv_recompute_threshold
):
logger.debug(
"Skipping remote pull for %s: %d remote tokens < threshold %d",
request.request_id,
count,
self.kv_recompute_threshold,
)
return 0, False
return count, True
# No remote prefill for this request.
return 0, False
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
params = request.kv_transfer_params
logger.debug(
"NIXLConnector update_state_after_alloc: "
"num_external_tokens=%s, kv_transfer_params=%s",
num_external_tokens,
params,
)
if not params:
return
if params.get("do_remote_decode") or (
params.get("do_remote_prefill") and self.is_bidirectional_kv_xfer_enabled
):
self._reqs_in_batch.add(request.request_id)
if self.use_host_buffer and params.get("do_remote_decode"):
# NOTE: when accelerator is not directly supported by Nixl,
# prefilled blocks need to be saved to host memory before transfer.
self._reqs_need_save[request.request_id] = request
elif params.get("do_remote_prefill") or (
params.get("do_remote_decode")
and self.is_bidirectional_kv_xfer_enabled
and not params.get("_remote_blocks_processed")
):
if params.get("remote_block_ids"):
if all(
p in params
for p in (
"remote_engine_id",
"remote_request_id",
"remote_host",
"remote_port",
)
):
# If remote_blocks and num_external_tokens = 0, we have
# a full prefix cache hit on the local node. We need to call
# send_notif in _read_blocks to free the memory on the remote node.
unhashed_local_block_ids: BlockIds = (
blocks.get_unhashed_block_ids_all_groups()
if num_external_tokens > 0
else ()
)
local_block_ids = self.get_exchange_clipped_blocks(
unhashed_local_block_ids
)
# Blocks covered by the local prefix cache, per KV cache group.
# Each count fixes where that group's DCP slice starts, which the
# worker needs to line up with the remote's slice.
local_num_computed_blocks = tuple(
sum(
block.block_hash is not None and not block.is_null
for block in group
)
for group in blocks.blocks
)
# Get unhashed blocks to pull from remote. Mind that a full prefix
# cache hit is indicated with an empty list.
self._reqs_need_recv[request.request_id] = (
request,
local_block_ids,
local_num_computed_blocks,
# Parked in WAITING_FOR_REMOTE_KVS only when there is
# something to pull; a full local hit stays RUNNING.
num_external_tokens > 0,
)
else:
logger.warning(
"Got invalid KVTransferParams: %s. This "
"request will not utilize KVTransfer",
params,
)
else:
assert num_external_tokens == 0
# Only trigger 1 KV transfer per request.
params["do_remote_prefill"] = False
params["_remote_blocks_processed"] = True
def request_finished(
self,
request: "Request",
block_ids: "BlockIds",
) -> tuple[bool, dict[str, Any] | None]:
"""Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
"""
from vllm.v1.request import RequestStatus
params = request.kv_transfer_params
logger.debug(
"NIXLConnector request_finished(%s), request_status=%s, "
"kv_transfer_params=%s",
request.request_id,
request.status,
params,
)
if not params:
return False, None
is_p_node = bool(params.get("do_remote_decode"))
is_d_node = not is_p_node
# Stop heartbeating for aborted requests that never reached finished_recving:
# normal path cleans up in update_connector_output.
self._stop_heartbeat(request.request_id)
if params.get("do_remote_prefill"):
# If do_remote_prefill is still True when the request is finished,
# update_state_after_alloc must not have been called (the request
# must have been aborted before it was scheduled, e.g. via the
# abort_immediately path used to clean up KV-transfer requests
# rejected at the D-side serving layer).
# To avoid stranding the prefill blocks in the prefill instance,
# we must add empty block_ids to _reqs_need_recv so that our
# worker side will notify and free blocks in the prefill instance.
self._reqs_need_recv[request.request_id] = (request, [], (), False)
params["do_remote_prefill"] = False
return False, None
if is_d_node and not self.is_bidirectional_kv_xfer_enabled:
return False, None
if request.status not in (
RequestStatus.FINISHED_LENGTH_CAPPED,
RequestStatus.FINISHED_STOPPED,
):
# Also include the case of a P/D Prefill request with immediate
# block free (eg abort). Stop tracking this request.
self._reqs_not_processed.add(request.request_id)
# Clear _reqs_need_save if a request is aborted as partial prefill.
self._reqs_need_save.pop(request.request_id, None)
return False, None
# TODO: check whether block_ids actually ever be 0. If not we could
# remove the conditional below
delay_free_blocks = any(len(group) > 0 for group in block_ids)
remote_num_tokens = 0
blocks_expiry_time = None
if delay_free_blocks:
# Prefill request on remote. It will be read from D upon completion
request_kv_blocks_ttl = self._kv_lease_duration
if is_d_node:
# For blocks pinned on D, use a simpler timeout for now instead of a
# lease mechanism as turn2 request is client-driven.
request_kv_blocks_ttl = self.decoder_kv_blocks_ttl
logger.debug(
"NIXLConnector request_finished(%s) waiting for %d seconds "
"before releasing blocks",
request.request_id,
request_kv_blocks_ttl,
)
self._reqs_need_send[request.request_id] = (
time.perf_counter() + request_kv_blocks_ttl
)
if is_d_node:
# D blocks expiry time exported for the turn-2 readback.
blocks_expiry_time = self._reqs_need_send[request.request_id]
# NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones),
# trimming down after allocating for the whole sequence length. Empty
# blocks are always at the start of the list.
# Here we "unpad" blocks to send the actual remote blocks to be read.
block_ids = self.get_exchange_clipped_blocks(block_ids)
remote_num_tokens = request.num_computed_tokens
return delay_free_blocks, dict(
do_remote_prefill=is_p_node,
do_remote_decode=is_d_node,
remote_block_ids=block_ids,
remote_engine_id=self.engine_id,
remote_request_id=request.request_id,
remote_host=self.side_channel_host,
remote_port=self.side_channel_port,
tp_size=self.transfer_tp_size,
dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size,
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
remote_num_tokens=remote_num_tokens,
remote_blocks_expiry_time=blocks_expiry_time,
transfer_mode=self._TRANSFER_MODE,
)
@@ -1,693 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pull-specific (READ) worker-side logic for the NIXL connector."""
import time
from typing import TYPE_CHECKING
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlConnectorMetadata,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
ReadSpec,
_is_attention_spec,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import KVCacheConfig
logger = init_logger(__name__)
# Slack (seconds) subtracted from D's exported block-expiry deadline on the turn-2
# readback, absorbing clock-offset error and read latency.
_KV_BLOCKS_EXPIRY_SAFETY_MARGIN = 5.0
class NixlPullConnectorWorker(NixlBaseConnectorWorker):
"""Pull-specific (READ) worker logic."""
def __init__(
self,
vllm_config: "VllmConfig",
engine_id: str,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, engine_id, kv_cache_config)
def start_load_kv(self, metadata: NixlConnectorMetadata):
"""Start loading by triggering non-blocking nixl_xfer.
We check for these trnxs to complete in each step().
"""
for req_id, meta in metadata.reqs_to_recv.items():
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
)
assert meta.remote is not None
# Remote block IDs are kept logical here; expanded in
# _read_blocks_for_req using the remote engine's phys ratio.
remote_engine_id = meta.remote.engine_id
logger.debug(
"start_load_kv for request %s from remote engine %s. "
"Num local_block_ids: %s. Num remote_block_ids: %s. ",
req_id,
remote_engine_id,
len(meta.local_physical_block_ids),
len(meta.remote.block_ids),
)
# Full local hits and aborted cleanup only notify P; no recv is awaited.
# On a full local hit:
# - Notification failure must not fail the request: its KV is local.
# - Receive completion must not be reported: num_external_tokens == 0,
# so the scheduler never entered WAITING_FOR_REMOTE_KVS to begin with.
# Aborted cleanup requests have already been removed from the scheduler.
if meta.awaiting_kvs or any(meta.local_block_ids):
self._recving_metadata[req_id] = meta
self._read_blocks_for_req(req_id, meta)
# Start transfers for requests whose handshakes have now finished.
while not self._ready_requests.empty():
self._read_blocks_for_req(*self._ready_requests.get_nowait())
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
# has nothing to send. Report the requests as sent right away so
# the scheduler-side aggregation (world_size workers, and any
# sibling connector inside a MultiConnector) still completes.
self._replicated_pcp_done_sending.update(metadata.reqs_to_send)
return
# Keep around the requests that have been part of a batch. This is
# needed because async scheduling pushes the misalignment between the
# moment in which requests expiration is set (P side) and the moment in
# which blocks are read from D. As P can now more easily lag behind D
# while processing the next batch, we make sure to only set an
# expiration for requests that have not been read from D yet.
for req_id in metadata.reqs_in_batch:
self._reqs_to_process.add(req_id)
# Remove all requests that are not to be processed (eg aborted).
for req_id in metadata.reqs_not_processed:
self._reqs_to_process.discard(req_id)
# We should never get an abort after setting an expiry timer
assert req_id not in self._reqs_to_send
# Add to requests that are waiting to be read and track expiration.
# Deadlines are stamped with the scheduler process's perf_counter,
# which is not comparable to ours when the worker runs in another
# process on another node (perf_counter epochs differ by boot time).
# Rebase the remaining TTL onto our clock; broadcast latency only
# lengthens the lease, which is the safe direction. A cross-node
# epoch gap larger than the TTL otherwise expires the lease on
# arrival and the blocks are freed before D reads them.
now_local = time.perf_counter()
for req_id, expiration_time in metadata.reqs_to_send.items():
if req_id in self._reqs_to_process:
if metadata.scheduler_clock:
expiration_time = now_local + (
expiration_time - metadata.scheduler_clock
)
self._reqs_to_send[req_id] = expiration_time
# Send heartbeats to P-side engines to keep KV blocks alive while
# requests sit in the D scheduler WAITING queue.
self._send_heartbeats(metadata)
def _is_turn2_read_expired(self, meta: ReqMeta) -> bool:
"""Whether D's cached blocks for this turn-2 readback have (nearly) expired."""
assert meta.remote is not None
blocks_expiry_time = meta.remote.blocks_expiry_time
# Deadline may be absent (router may not forward it) -> read as usual.
if blocks_expiry_time is None or not meta.local_physical_block_ids:
return False
clock_offset = self._engine_clock_offset[meta.remote.engine_id]
deadline = blocks_expiry_time - clock_offset
return time.perf_counter() + _KV_BLOCKS_EXPIRY_SAFETY_MARGIN >= deadline
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
assert meta.remote is not None and self.transfer_topo is not None
engine_id = meta.remote.engine_id
if engine_id in self._invalid_remote_engines:
self._handle_failed_transfer(
req_id, None, self._recv_failures, record_failed_transfer=False
)
return
if engine_id not in self._remote_agents:
# Also re-handshakes queued requests that outlived invalid-peer cleanup.
self._background_nixl_handshake(req_id, engine_id, meta)
return
# Update last activity from this remote. Mind that cleanup is done on main
# thread (this one), so we don't race on this structure.
self._engine_last_active[engine_id] = time.perf_counter()
if self._bidirectional_kv_xfer_enabled and self._is_turn2_read_expired(meta):
logger.warning(
"Declining expired remote read for %s from engine %s.",
req_id,
engine_id,
)
self.xfer_stats.record_kv_expired_req()
# KV expiry is reported separately from transport failures, so only
# the state cleanup side of _handle_failed_transfer runs here.
self._handle_failed_transfer(
req_id, None, self._recv_failures, record_failed_transfer=False
)
return
if any(len(group) > 0 for group in meta.local_block_ids):
# The scheduler waits for finished_recving from *every* worker.
# Under DCP a rank's slice can legitimately come out empty when its
# interleaved positions fall past the end of the sequence. _read_blocks
# then takes the notify-only path without registering a transfer.
# Seed the entry so this rank still reports completion.
self._recving_transfers.setdefault(req_id, [])
plan = self.tp_mappings[engine_id]
remote_info = self.transfer_topo.get_engine_info(engine_id)
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
dcp_active = self.dcp_size > 1 or remote_info.remote_dcp_size > 1
local_block_ids = meta.local_physical_block_ids
remote_region_groups = self.dst_region_group_ids[engine_id]
local_region_groups = self.region_group_ids or remote_region_groups
if not local_block_ids:
# Region expansion cannot index empty groups. Pass empty specs to
# _read_blocks so its existing cache-hit notification path runs.
read_specs = [
ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[])
for rank in plan.all_source_ranks
]
elif local_region_groups != remote_region_groups:
if not self.use_mla or self._has_mamba:
raise NotImplementedError(
"Different NIXL cache-group layouts are only supported for "
"pure MLA models"
)
if self.block_size != remote_info.remote_block_size:
raise NotImplementedError(
"Region-mapped NIXL transfers require matching physical block sizes"
)
remote_physical_block_ids = self._logical_to_kernel_block_ids(
meta.remote.block_ids,
remote_info.remote_physical_blocks_per_logical,
)
remote_by_region = self._block_ids_by_region(
remote_physical_block_ids, remote_region_groups
)
local_by_region = self._block_ids_by_region(
local_block_ids, local_region_groups
)
num_computed_blocks = None
num_remote_blocks = None
if (
meta.remote.num_tokens is not None
and meta.local_num_computed_blocks
and all(group >= 0 for group in local_region_groups)
and all(group >= 0 for group in remote_region_groups)
):
transfer_groups = self.kv_cache_config.transfer_group_ids
num_computed_blocks = [
meta.local_num_computed_blocks[transfer_groups[group]]
* self._physical_blocks_per_logical_kv_block
for group in local_region_groups
]
num_remote_blocks = cdiv(
meta.remote.num_tokens, remote_info.remote_block_size
)
elif any(local_by_region) and (
dcp_active
or remote_info.remote_physical_blocks_per_logical
!= self._physical_blocks_per_logical_kv_block
):
raise NotImplementedError(
"Region-mapped pulls with DCP or different logical block sizes "
"require remote_num_tokens, per-group prefix counts "
"and unshared regions"
)
read_specs = [
ReadSpec(
rank,
*self._apply_prefix_caching_by_region(
local_by_region,
remote_by_region,
num_computed_blocks=num_computed_blocks,
num_remote_blocks=num_remote_blocks,
remote_rank=rank,
remote_dcp_size=remote_info.remote_dcp_size,
),
block_ids_by_region=True,
)
for rank in plan.all_source_ranks
]
meta.region_blocks_to_zero = [
list(blocks[sum(len(spec.local_block_ids[r]) for spec in read_specs) :])
for r, blocks in enumerate(local_by_region)
]
else:
remote_logical_block_ids = meta.remote.block_ids
meta.remote.block_ids = self._logical_to_kernel_block_ids(
remote_logical_block_ids,
remote_info.remote_physical_blocks_per_logical,
)
num_groups = len(meta.local_block_ids)
def group_ids(block_ids: BlockIds, rank: int) -> list[list[int]]:
return [
list(block_ids[g]) if rank in plan.source_ranks_per_group[g] else []
for g in range(num_groups)
]
read_specs = []
for rank in plan.all_source_ranks:
if dcp_active:
local_ids = group_ids(meta.local_block_ids, rank)
remote_ids = group_ids(remote_logical_block_ids, rank)
for g in range(num_groups):
if not local_ids[g] or not _is_attention_spec(
self._group_spec_types[g]
):
continue
local_ids[g], remote_ids[g] = self._apply_dcp_prefix_caching(
local_ids[g],
remote_ids[g],
remote_rank=rank,
local_dcp_size=self.dcp_size,
local_dcp_rank=self.dcp_rank,
remote_dcp_size=remote_info.remote_dcp_size,
local_num_computed_blocks=(
meta.local_num_computed_blocks[g]
),
)
local_physical_ids = self._logical_to_kernel_block_ids(
local_ids, self._physical_blocks_per_logical_kv_block
)
remote_physical_ids = self._logical_to_kernel_block_ids(
remote_ids,
remote_info.remote_physical_blocks_per_logical,
)
else:
local_physical_ids = group_ids(meta.local_physical_block_ids, rank)
remote_physical_ids = group_ids(meta.remote.block_ids, rank)
read_specs.append(
ReadSpec(
remote_rank=rank,
local_block_ids=local_physical_ids,
remote_block_ids=remote_physical_ids,
)
)
# D may have to perform multiple reads from different remote ranks.
# Pure MLA reads once because its cache is replicated. Hybrid
# MLA+SSM still needs one read per SSM source rank. With DCP, pure
# MLA may also read from multiple ranks (disjoint token slices).
if self.use_mla and tp_ratio < 0 and not self._has_mamba and not dcp_active:
assert len(read_specs) == 1
for i, spec in enumerate(read_specs):
remote_block_size = remote_info.remote_block_size
logger.debug(
"Remote agent %s available, calling _read_blocks"
" on remote rank %s with remote block size %s for req %s",
meta.remote.engine_id,
spec.remote_rank,
remote_block_size,
req_id,
)
# Get side handles.
if tp_ratio < 0 and (not self.use_mla or len(read_specs) > 1):
# Remote tp_size > local tp_size: we must perform multiple
# reads. Get the memory chunk onto which we will write to.
split_key = (tp_ratio, remote_block_size)
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
local_dram_handle = (
self._dram_src_handles_by_tp_ratio[split_key][i]
if self._mixed_mem_types
else None
)
else:
# Single read from remote, we write to the whole memory region.
# Also handle remote block size different from local block size.
local_xfer_side_handle = self.src_xfer_handles_by_block_size[
remote_block_size
]
local_dram_handle = (
self._dram_src_handles_by_block_size[remote_block_size]
if self._mixed_mem_types
else None
)
# Destination handle: remote_engine_id -> remote_rank -> handle.
remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][
spec.remote_rank
]
# Once a read routes the request to failure reporting, the
# scheduler may free and reuse its blocks, so no sibling READs
# may be posted (and P must not be notified).
if not self._read_blocks(
read_spec=spec,
request_id=req_id,
dst_engine_id=meta.remote.engine_id,
remote_request_id=meta.remote.request_id,
local_xfer_side_handle=local_xfer_side_handle,
local_dram_handle=local_dram_handle,
remote_xfer_side_handle=remote_xfer_side_handle,
expected_consumers=plan.local_consumers,
awaiting_kvs=meta.awaiting_kvs,
):
return
if self.use_mla and tp_ratio < 0 and len(read_specs) == 1:
# ..but we still need to notify the other remote ranks that we
# have the blocks we need so they can update the request state.
# Same thing for DCP (tp_size == dcp_size), so the raw tp_ratio already
# reflects whether any remote replica is left unchosen.
notif_id = f"{meta.remote.request_id}:{plan.local_consumers}".encode()
remote_agents = self._remote_agents[meta.remote.engine_id]
for rank_to_notify, agent in remote_agents.items():
if rank_to_notify != (0, read_specs[0].remote_rank):
try:
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
except Exception as e:
self._log_failure(
failure_type="notification_failed",
msg="Remote rank will not update request state. "
"This may indicate network issues.",
req_id=req_id,
error=e,
dst_engine_id=meta.remote.engine_id,
remote_rank=rank_to_notify[1],
remote_agent_name=agent,
)
self.xfer_stats.record_failed_notification()
def _read_blocks(
self,
read_spec: ReadSpec,
dst_engine_id: str,
request_id: str,
remote_request_id: str,
local_xfer_side_handle: int,
local_dram_handle: int | None,
remote_xfer_side_handle: int,
expected_consumers: int,
awaiting_kvs: bool,
) -> bool:
"""Post a READ point-to-point xfer request from a single local worker to
a single remote worker.
Returns True when the read was posted (or was unnecessary), False
when the request was routed to failure reporting — the caller must
not post further transfers for it.
"""
assert self.transfer_topo is not None
remote_rank = read_spec.remote_rank
local_block_ids = read_spec.local_block_ids
remote_block_ids = read_spec.remote_block_ids
remote_info = self.transfer_topo.get_engine_info(dst_engine_id)
block_size_ratio = self.transfer_topo.block_size_ratio(
remote_info.remote_block_size
)
if block_size_ratio > 1:
if read_spec.block_ids_by_region:
raise NotImplementedError(
"Region-mapped NIXL transfers require matching physical block sizes"
)
local_block_ids, remote_block_ids = (
self._map_block_ids_for_block_size_ratio(
local_block_ids, remote_block_ids, block_size_ratio
)
)
# NOTE(rob): having the staging blocks be on the READER side is
# not going to work well (since we will have to call rearrange tensors).
# after we detect the txn is complete (which means we cannot make the
# read trxn async easily). If we want to make "READ" happen cleanly,
# then we will need to have the staging blocks on the remote side.
# NOTE(rob): according to nvidia the staging blocks are used to
# saturate IB with heterogeneous TP sizes.
# Number of local workers that will notify this producer worker.
# Propagate on notification so dst worker can wait before freeing.
notif_id = f"{remote_request_id}:{expected_consumers}".encode()
# Full prefix cache hit: do not need to read remote blocks,
# just notify P worker that we have the blocks we need.
if not any(len(group) > 0 for group in local_block_ids):
# A full prefix cache hit is indicated with an empty list.
agent_name = self._remote_agents[dst_engine_id][(0, remote_rank)]
try:
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id)
except Exception as e:
self._log_failure(
failure_type="notification_failed",
msg="P worker blocks will be freed after timeout. "
"This may indicate network issues.",
req_id=request_id,
error=e,
dst_engine_id=dst_engine_id,
remote_rank=remote_rank,
remote_agent_name=agent_name,
)
self.xfer_stats.record_failed_notification()
# Report even on notification failure: the KV is already local, and
# an unreported parked request would hold its blocks forever.
# Notify-only recvs must stay unreported (scheduler asserts).
if awaiting_kvs:
self._recving_transfers.setdefault(request_id, [])
return True
if not read_spec.block_ids_by_region:
assert (
len(remote_block_ids)
== len(local_block_ids)
== len(self.kv_cache_config.transfer_groups)
)
if not (self.dcp_size > 1 or remote_info.remote_dcp_size > 1):
local_block_ids, remote_block_ids = self._apply_prefix_caching(
decode_block_ids=local_block_ids,
prefill_block_ids=remote_block_ids,
decode_physical_per_logical=(
self._physical_blocks_per_logical_kv_block
),
prefill_physical_per_logical=(
remote_info.remote_physical_blocks_per_logical
),
)
# NOTE (nicolo) With homogeneous TP, each TP worker loads KV from
# corresponding rank. With heterogeneous TP, fixing D>P, the D tp
# workers will issue xfers to parts of the P worker remote kv caches.
# Get descs ids.
remote_block_descs_ids = self._compute_desc_ids(
block_ids=remote_block_ids,
dst_num_blocks=self.dst_num_blocks[dst_engine_id],
block_size_ratio=None,
physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical,
region_num_blocks=(self.dst_region_num_blocks.get(dst_engine_id) or None),
region_group_ids=(
list(range(self.num_regions))
if read_spec.block_ids_by_region
else (self.dst_region_group_ids.get(dst_engine_id) or None)
),
uses_region_group_mapping=(
self.num_regions > 1
if read_spec.block_ids_by_region
else self.dst_uses_region_group_mapping[dst_engine_id]
),
)
local_block_descs_ids = self._compute_desc_ids(
block_ids=local_block_ids,
dst_num_blocks=self.dst_num_blocks[self.engine_id],
block_size_ratio=block_size_ratio,
physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block,
region_num_blocks=(self.dst_region_num_blocks.get(self.engine_id) or None),
region_group_ids=(
list(range(self.num_regions))
if read_spec.block_ids_by_region
else (self.region_group_ids or None)
),
uses_region_group_mapping=(
self.num_regions > 1
if read_spec.block_ids_by_region
else self._uses_region_group_mapping
),
)
assert len(local_block_descs_ids) == len(remote_block_descs_ids)
# Prepare transfer with Nixl.
handle = None
try:
if self._mixed_mem_types:
self._read_blocks_mixed(
request_id=request_id,
local_block_size_key=remote_info.remote_block_size,
local_device_handle=local_xfer_side_handle,
local_dram_handle=local_dram_handle,
remote_xfer_side_handle=remote_xfer_side_handle,
local_block_descs_ids=local_block_descs_ids,
remote_block_descs_ids=remote_block_descs_ids,
notif_agent=self._remote_agents[dst_engine_id][(0, remote_rank)],
notif_id=notif_id,
)
return True
handle = self.nixl_wrapper.make_prepped_xfer(
"READ",
local_xfer_side_handle,
local_block_descs_ids,
remote_xfer_side_handle,
remote_block_descs_ids,
notif_msg=notif_id,
)
# Begin async xfer.
self.nixl_wrapper.transfer(handle)
# Use handle to check completion in future step().
self._recving_transfers[request_id].append(handle)
return True
except Exception as e:
self._log_failure(
failure_type="transfer_setup_failed",
req_id=request_id,
msg="Deferring failure reporting until outstanding transfers finish",
error=e,
dst_engine_id=dst_engine_id,
remote_rank=remote_rank,
)
if not self._handle_failed_transfer(
request_id, handle, self._recv_failures
):
assert handle is not None
self._recving_transfers[request_id].append(handle)
return False
def _read_blocks_mixed(
self,
request_id: str,
local_block_size_key: int,
local_device_handle: int,
local_dram_handle: int | None,
remote_xfer_side_handle: int,
local_block_descs_ids: np.ndarray,
remote_block_descs_ids: np.ndarray,
notif_agent: str,
notif_id: bytes,
) -> None:
"""Split a READ across the local DRAM and device descriptor lists."""
desc_is_dram = self._desc_is_dram_by_block_size[local_block_size_key]
desc_pos = self._desc_pos_by_block_size[local_block_size_key]
local_ids = np.asarray(local_block_descs_ids)
remote_ids = np.asarray(remote_block_descs_ids)
is_dram = desc_is_dram[local_ids]
if self._skip_dram_xfer:
# TP rank 0 reads the DRAM (shared host pool) part for every
# local rank; keep only the device part. Register the request
# even if nothing is left, so it still completes and notifies.
keep = ~is_dram
local_ids = local_ids[keep]
remote_ids = remote_ids[keep]
is_dram = is_dram[keep]
self._recving_transfers.setdefault(request_id, [])
assert local_dram_handle is not None or not is_dram.any()
reads = (
(is_dram, local_dram_handle),
(~is_dram, local_device_handle),
)
handles: list[int] = []
try:
for mask, local_handle in reads:
if mask.any():
handles.append(
self.nixl_wrapper.make_prepped_xfer(
"READ",
local_handle,
desc_pos[local_ids[mask]],
remote_xfer_side_handle,
remote_ids[mask],
)
)
except Exception:
for handle in handles:
if not self._try_release_xfer_handle(request_id, handle):
self._recving_transfers[request_id].append(handle)
raise
self._pending_recv_notifs.setdefault(request_id, []).append(
(notif_agent, notif_id)
)
for i, handle in enumerate(handles):
try:
self.nixl_wrapper.transfer(handle)
except Exception:
for unstarted in handles[i:]:
if not self._try_release_xfer_handle(request_id, unstarted):
self._recving_transfers[request_id].append(unstarted)
raise
self._recving_transfers[request_id].append(handle)
def _get_new_notifs(self) -> set[str]:
"""Get req_ids which got a remote xfer message. When multiple consumers
are reading from the same producer (heterogeneous TP or DCP
scenario), wait for all consumers to be done pulling.
Also handles heartbeat notifications ("HB:req1,req2,...") by
extending the lease on the referenced requests.
"""
assert self.transfer_topo is not None
notified_req_ids: set[str] = set()
for notifs in self.nixl_wrapper.get_new_notifs().values():
for notif in notifs:
msg = notif.decode("utf-8")
# Handle heartbeat messages from D-side.
if msg.startswith("HB:"):
self._handle_heartbeat(msg[3:])
continue
req_id, expected_consumers = msg.rsplit(":", 1)
if (
req_id not in self._reqs_to_send
and req_id not in self._reqs_to_process
):
self.xfer_stats.record_notification_after_expiry()
logger.error(
"Potentially invalid KV blocks for "
"unrecognized request %s were retrieved by "
"a decode worker. They may have expired.",
req_id,
)
continue
# Every reader of this req_id reports the same count (it's
# derived from aggregate topology, not the specific rank),
# so repeated notifications never disagree on it.
self.expected_consumer_notifications_by_req[req_id] = int(
expected_consumers
)
self.consumer_notification_counts_by_req[req_id] += 1
# Wait all consumers (D) to be done reading before freeing.
if (
self.consumer_notification_counts_by_req[req_id]
== self.expected_consumer_notifications_by_req[req_id]
):
notified_req_ids.add(req_id)
del self.consumer_notification_counts_by_req[req_id]
del self.expected_consumer_notifications_by_req[req_id]
self._reqs_to_process.remove(req_id)
self._reqs_to_send.pop(req_id, None)
return notified_req_ids
@@ -1,357 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Push-specific scheduler-side logic for the NIXL connector.
In push mode, scheduler-side responsibilities are:
* D side (decode): on ``update_state_after_alloc``, stash registration data
(D's identity + locally allocated block IDs) into
``_push_pending_registrations``. The D worker drains it from
``meta.push_registrations`` next step and sends a NIXL notification to the
P worker (no scheduler-level networking).
* P side (prefill): on ``request_finished``, stash the finished block IDs
into ``_finished_request_blocks`` for the lease, and into
``_newly_finished_push_blocks`` so the P worker picks them up via
``meta.push_finished_blocks`` and matches against any D registrations
it already received via NIXL notifications.
* Both sides: ``has_pending_push_work`` keeps the engine main loop stepping
while pushes are in flight. ``update_connector_output`` cleans up
``_finished_request_blocks`` once the WRITE completes.
A soft per-registration watchdog on the D scheduler fails requests that have
been registered but not fulfilled within a configurable timeout.
"""
from __future__ import annotations
import time
from typing import TYPE_CHECKING, Any
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
NixlBaseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlConnectorMetadata,
ReqId,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
"""Push-specific scheduler logic (WRITE-based KV transfer).
All P2P communication is deferred to the worker level via NIXL
notifications. The scheduler communicates with workers only through
the standard ``build_connector_meta`` / ``update_connector_output``
hooks.
"""
_TRANSFER_MODE: str = "push"
def __init__(
self,
vllm_config: VllmConfig,
engine_id: str,
kv_cache_config: KVCacheConfig,
):
super().__init__(vllm_config, engine_id, kv_cache_config)
if self.is_bidirectional_kv_xfer_enabled:
raise NotImplementedError(
"Bidirectional KV transfer is not supported for NIXL push connector."
)
# D-side: registration data to pass to D workers via metadata on
# the next ``build_connector_meta`` call.
self._push_pending_registrations: dict[ReqId, dict[str, Any]] = {}
# D-side: track the wall-clock deadline for each registered request
# to detect "registered but never fulfilled" failures (e.g. the P
# node disappeared after registration). Keyed by D request_id.
self._push_registration_deadlines: dict[ReqId, float] = {}
# P-side: block IDs for finished requests, kept for the lease and
# used to drive ``has_pending_push_work``.
self._finished_request_blocks: dict[ReqId, BlockIds] = {}
# P-side: newly finished blocks to ship to P workers on next step.
self._newly_finished_push_blocks: dict[ReqId, BlockIds] = {}
# Soft watchdog timeout (seconds) for D-side registrations that
# never receive a push completion. Defaults to the existing
# decoder KV blocks TTL so behaviour matches the lease.
assert vllm_config.kv_transfer_config is not None
self._push_registration_timeout: float = float(
vllm_config.kv_transfer_config.get_from_extra_config(
"push_registration_timeout",
self.decoder_kv_blocks_ttl,
)
)
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int, bool]:
"""In push mode, D doesn't pull — it registers blocks and waits.
However, we still need to handle the do_remote_prefill case where D
needs to know how many tokens will be pushed.
"""
params = request.kv_transfer_params
logger.debug(
"NixlPushConnector get_num_new_matched_tokens: "
"num_computed_tokens=%s, kv_transfer_params=%s",
num_computed_tokens,
params,
)
if params is not None and params.get("do_remote_prefill"):
token_ids = request.prompt_token_ids or []
actual = self._get_remote_prefill_token_count(len(token_ids))
count = actual - num_computed_tokens
if count > 0:
return count, True
return 0, False
def update_state_after_alloc(
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
):
"""In push mode, D stores registration data for the worker to send
to P via NIXL notification (deferred to ``build_connector_meta``).
"""
params = request.kv_transfer_params
logger.debug(
"NixlPushConnector update_state_after_alloc: "
"num_external_tokens=%s, kv_transfer_params=%s",
num_external_tokens,
params,
)
if not params:
return
# P side: track the request as in-batch so the lease accounting
# matches what the worker expects on the next step.
if params.get("do_remote_decode"):
self._reqs_in_batch.add(request.request_id)
# P side with host-buffer offload: defer save to the worker.
if self.use_host_buffer and params.get("do_remote_decode"):
self._reqs_need_save[request.request_id] = request
return
# D side: only act on the first call (``do_remote_prefill`` is
# unset on re-entry by the marker below).
if not params.get("do_remote_prefill"):
return
if num_external_tokens <= 0:
# Nothing to receive: full prefix-cache hit on D, no
# registration to stage.
return
# First-pass D path: stash registration data the worker will
# ship to P on the next ``build_connector_meta`` cycle.
logger.debug(
"KV PUSH mode: D node storing registration for request %s",
request.request_id,
)
local_block_ids: BlockIds = blocks.get_unhashed_block_ids_all_groups()
local_block_ids = self.get_exchange_clipped_blocks(local_block_ids)
# ``remote_*`` fields are P's coordinates (from D's perspective).
# ``decode_*`` fields are D's own info that P needs for the
# reverse handshake before WRITE-ing.
self._push_pending_registrations[request.request_id] = {
"request_id": request.request_id,
"decode_engine_id": self.engine_id,
"decode_host": self.side_channel_host,
"decode_port": self.side_channel_port,
"decode_tp_size": (self.vllm_config.parallel_config.tensor_parallel_size),
"local_block_ids": local_block_ids,
"remote_engine_id": params["remote_engine_id"],
"remote_host": params["remote_host"],
"remote_port": params["remote_port"],
"remote_tp_size": params["tp_size"],
"remote_pp_size": params.get("pp_size", 1),
}
self._push_registration_deadlines[request.request_id] = (
time.perf_counter() + self._push_registration_timeout
)
# In push mode D doesn't know P's blocks; P determines them
# from the registration. We still track the request as
# needing recv so the engine waits for P's WRITE completion.
# ``remote_block_ids`` is also seeded to an empty tuple so the
# base scheduler's ``add_new_req_to_recv`` can build the
# ReqMeta without a KeyError — the actual remote block IDs are
# learned by P over the NIXL handshake at WRITE time.
params["remote_block_ids"] = ()
self._reqs_need_recv[request.request_id] = (request, local_block_ids, (), False)
# Mark as processed so a re-entry (e.g. preemption + reschedule)
# doesn't re-stage the registration.
params["do_remote_prefill"] = False
def request_finished(
self,
request: Request,
block_ids: BlockIds,
) -> tuple[bool, dict[str, Any] | None]:
"""Push-mode request_finished: stores blocks for workers."""
from vllm.v1.request import RequestStatus
params = request.kv_transfer_params
logger.debug(
"NixlPushConnector request_finished(%s), request_status=%s, "
"kv_transfer_params=%s",
request.request_id,
request.status,
params,
)
if not params:
return False, None
is_p_node = bool(params.get("do_remote_decode"))
self._stop_heartbeat(request.request_id)
# Drop any pending registration deadline; the request either
# completed or was cancelled.
self._push_registration_deadlines.pop(request.request_id, None)
if params.get("do_remote_prefill"):
# ``do_remote_prefill`` is still set, which means
# ``update_state_after_alloc`` never ran (it would have
# flipped this flag to False). The request was aborted
# before it could be scheduled — e.g. rejected at the D
# serving layer via abort_immediately. To keep P from
# stranding the prefill blocks, we still register an empty
# recv so the worker emits a notif that lets P free them.
# Seed remote_block_ids so add_new_req_to_recv won't KeyError.
params["remote_block_ids"] = ()
self._reqs_need_recv[request.request_id] = (request, [], (), False)
params["do_remote_prefill"] = False
return False, None
# Push connector only acts on the P-side terminal path; D-side
# finishing without a remote prefill is a no-op.
if not is_p_node:
return False, None
if request.status not in (
RequestStatus.FINISHED_LENGTH_CAPPED,
RequestStatus.FINISHED_STOPPED,
):
self._reqs_not_processed.add(request.request_id)
self._reqs_need_save.pop(request.request_id, None)
return False, None
delay_free_blocks = any(len(group) > 0 for group in block_ids)
remote_num_tokens = 0
if delay_free_blocks:
logger.debug(
"NixlPushConnector request_finished(%s) waiting for %d seconds "
"before releasing blocks",
request.request_id,
self._kv_lease_duration,
)
self._reqs_need_send[request.request_id] = (
time.perf_counter() + self._kv_lease_duration
)
block_ids = self.get_exchange_clipped_blocks(block_ids)
remote_num_tokens = request.num_computed_tokens
# Store finished blocks for worker-level matching with D
# registrations (via NIXL notifications).
self._finished_request_blocks[request.request_id] = block_ids
self._newly_finished_push_blocks[request.request_id] = block_ids
return delay_free_blocks, dict(
do_remote_prefill=True,
do_remote_decode=False,
remote_block_ids=block_ids,
remote_engine_id=self.engine_id,
remote_request_id=request.request_id,
remote_host=self.side_channel_host,
remote_port=self.side_channel_port,
tp_size=self.transfer_tp_size,
dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size,
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
remote_num_tokens=remote_num_tokens,
transfer_mode=self._TRANSFER_MODE,
)
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
meta = super().build_connector_meta(scheduler_output)
assert isinstance(meta, NixlConnectorMetadata)
# Watchdog: any D-side registration whose deadline has passed without
# a corresponding push completion is treated as failed and cleaned up.
# The corresponding request is already tracked via _reqs_need_recv;
# the engine layer will eventually time it out via the lease, but we
# at least drop the stale registration so we don't keep retrying.
now = time.perf_counter()
# Deadlines are inserted in non-decreasing order (monotonic clock +
# constant timeout, armed once per request), and dict insertion order
# is preserved across key deletions, so we can stop at the first
# not-yet-expired entry instead of scanning the whole dict.
expired = []
for rid, deadline in self._push_registration_deadlines.items():
if deadline > now:
break
expired.append(rid)
for rid in expired:
self._push_registration_deadlines.pop(rid, None)
# Avoid resending a registration that already timed out.
self._push_pending_registrations.pop(rid, None)
logger.warning(
"NixlPushConnector: registration for request %s timed out "
"after %.1fs without a push completion",
rid,
self._push_registration_timeout,
)
# D side: package pending registrations for D workers to send out.
if self._push_pending_registrations:
meta.push_registrations = dict(self._push_pending_registrations)
self._push_pending_registrations.clear()
# P side: package newly finished blocks for P workers to match against
# any D registrations they have received via NIXL notifications.
if self._newly_finished_push_blocks:
meta.push_finished_blocks = dict(self._newly_finished_push_blocks)
self._newly_finished_push_blocks.clear()
return meta
def has_pending_push_work(self) -> bool:
# Keep the engine main loop alive while we have:
# - finished P blocks awaiting WRITE completion, or
# - pending D registrations the worker has not yet shipped, or
# - newly finished blocks not yet shipped to P workers.
return bool(self._finished_request_blocks or self._push_pending_registrations)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Clean up finished request blocks after push completes."""
super().update_connector_output(connector_output)
for req_id in connector_output.finished_sending or ():
self._finished_request_blocks.pop(req_id, None)
# On D side, finished_recving means the push completed; clear the
# watchdog so we don't trip an expiration on a fulfilled request.
for req_id in connector_output.finished_recving or ():
self._push_registration_deadlines.pop(req_id, None)
@@ -1,827 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Push-specific (WRITE) worker-side logic for the NIXL connector.
A dedicated ``nixl-push-writer`` thread owns all push-related NIXL ops:
calls ``get_new_notifs`` (routing PUSH_REG internally; HB / completion
notifs are forwarded to the engine main thread), sends PUSH_REG via
``send_notif``, matches D registrations with P finished blocks, and
issues WRITE transfers via ``make_prepped_xfer`` / ``transfer``.
The engine main thread feeds the writer through queues:
``_reg_send_inbox`` (D-side regs to send), ``_finished_blocks_inbox``
(P-side blocks from metadata) and ``_pending_completion_notifs``
(non-PUSH_REG notifs forwarded back for HB / completion accounting).
The handshake-completion callback feeds ``_deferred_push_inbox`` with
matched pushes whose P→D handshake has finished so the writer can
(re-)issue the WRITE without ever blocking on the network.
Wake model: the writer self-polls every
``_PUSH_WRITER_POLL_INTERVAL_MS`` only while it has unmatched
``_push_finished_blocks`` (i.e. P-side blocks waiting for a D PUSH_REG
notif that has no other wake source). All other progress is
event-driven: the engine main thread sets ``_push_writer_wake`` from
``start_load_kv`` (when handing it new work) and from ``get_finished``
(so each engine step gives the writer a chance to drain NIXL notifs);
the handshake-completion callback sets the same event after a deferred
PUSH_REG send or a deferred push WRITE has been queued. When a request's
lease expires (the base worker reports it via ``done_sending``) or the WRITE completes,
``get_finished`` enqueues an eviction onto ``_evict_finished_inbox`` so
the writer drops any leftover ``_push_finished_blocks`` /
``_pending_d_registrations`` and stops self-polling.
"""
import queue
import threading
import time
from collections import defaultdict
from concurrent.futures import Future
from typing import TYPE_CHECKING, Any
import msgspec
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
PUSH_REG_NOTIF_PREFIX,
NixlConnectorMetadata,
RemoteMeta,
ReqId,
ReqMeta,
TransferHandle,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
ReadSpec,
_is_attention_spec,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import get_base_request_id
from vllm.logger import init_logger
if TYPE_CHECKING:
import torch
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import KVCacheConfig
logger = init_logger(__name__)
# Writer-thread poll cadence while there is in-flight push state. When
# fully idle, the writer blocks on a wake event signalled by the engine
# main thread (start_load_kv / get_finished). Smaller -> lower latency
# while active, slightly more CPU.
_PUSH_WRITER_POLL_INTERVAL_MS = 1.0
class NixlPushConnectorWorker(NixlBaseConnectorWorker):
"""Push-specific (WRITE) worker logic. See module docstring."""
# Distinguishes push from pull in the NIXL compatibility hash.
_TRANSFER_MODE: str = "push"
_supports_pp_hma = True
def __init__(
self,
vllm_config: "VllmConfig",
engine_id: str,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, engine_id, kv_cache_config)
# Heartbeat handshakes to a PP-sharded producer must be notif-only,
# like the PUSH_REG path.
self._hb_handshake_notif_only = True
# Push-specific state.
# P-side: outgoing WRITE handles awaiting completion, keyed by
# request_id. Mutated by writer (submit) and main thread
# (``_pop_done_transfers``); guarded by
# ``_sending_transfers_lock``.
self._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list)
self._send_failures: set[ReqId] = set()
self._sending_transfers_lock = threading.Lock()
# Writer-thread owned matching state.
# P-side: finished request blocks received from scheduler metadata
# that have not yet been matched with an incoming D registration.
self._push_finished_blocks: dict[ReqId, BlockIds] = {}
# P-side: D registrations received via NIXL notification that have
# not yet been matched with a finished P request.
self._pending_d_registrations: dict[ReqId, dict[str, Any]] = {}
# Cross-thread channels.
self._reg_send_inbox: queue.Queue[tuple[str, dict[str, Any]]] = queue.Queue()
self._finished_blocks_inbox: queue.Queue[tuple[str, BlockIds]] = queue.Queue()
self._pending_completion_notifs: queue.Queue[bytes] = queue.Queue()
# Main thread → writer: req_ids whose lease has expired or whose
# WRITE has completed. Writer drops them from
# ``_push_finished_blocks`` so an unmatched entry doesn't keep the
# writer busy-polling forever.
self._evict_finished_inbox: queue.Queue[str] = queue.Queue()
# Handshakes that have just completed and are ready for the WRITE on wthread
self._deferred_push_inbox = queue.Queue[tuple[str, BlockIds, dict[str, Any]]]()
# Wake signal from engine main thread (start_load_kv / get_finished).
# Writer self-polls at _PUSH_WRITER_POLL_INTERVAL_MS while it has
# active in-flight state; otherwise it blocks until signalled.
self._push_writer_wake = threading.Event()
self._push_writer_stop = threading.Event()
self._push_writer_thread: threading.Thread | None = None
# --- Lifecycle ----------------------------------------------------- #
def register_kv_caches(self, kv_caches: dict[str, "torch.Tensor"]):
super().register_kv_caches(kv_caches)
if self._mixed_mem_types:
raise NotImplementedError(
"NixlPushConnector does not support mixed-memory KV caches"
)
if self._push_writer_thread is None:
self._push_writer_thread = threading.Thread(
target=self._push_writer_loop,
daemon=True,
name="nixl-push-writer",
)
self._push_writer_thread.start()
logger.info("nixl-push-writer thread started (rank=%d)", self.tp_rank)
def shutdown(self) -> None:
self._push_writer_stop.set()
# Unblock the writer if it's waiting in the no-active-state branch.
self._push_writer_wake.set()
if self._push_writer_thread is not None:
self._push_writer_thread.join(timeout=2)
self._push_writer_thread = None
with self._sending_transfers_lock:
for handles in self._sending_transfers.values():
for handle in handles:
self.nixl_wrapper.release_xfer_handle(handle)
self._sending_transfers.clear()
self._send_failures.clear()
super().shutdown()
# --- Engine-main-thread entry point -------------------------------- #
def start_load_kv(self, metadata: NixlConnectorMetadata):
"""Pre-process metadata; defer NIXL ops to the writer thread."""
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
return
# D-side: track reqs waiting for P to push.
for req_id, meta in metadata.reqs_to_recv.items():
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
)
assert meta.remote is not None
remote_engine_id = meta.remote.engine_id
# Update last activity from this remote (same as pull), but only for
# an already-connected engine so we never leave an _engine_last_active
# entry without a _remote_agents entry (breaks _cleanup_remote_engine).
if remote_engine_id in self._remote_agents:
self._engine_last_active[remote_engine_id] = time.perf_counter()
logger.debug(
"start_load_kv (push) for request %s from remote engine %s. "
"Num local_block_ids: %s. Num remote_block_ids: %s. ",
req_id,
remote_engine_id,
len(meta.local_physical_block_ids),
len(meta.remote.block_ids),
)
self._recving_metadata[req_id] = meta
# --- D-side: registrations to send to P via NIXL ---
if metadata.push_registrations:
for req_id, reg_data in metadata.push_registrations.items():
self._reg_send_inbox.put((req_id, reg_data))
self._push_writer_wake.set()
# --- P-side: newly finished blocks awaiting a D registration match ---
if metadata.push_finished_blocks:
for req_id, block_ids in metadata.push_finished_blocks.items():
self._finished_blocks_inbox.put((req_id, block_ids))
self._push_writer_wake.set()
# Batch + lease tracking (same as pull).
for req_id in metadata.reqs_in_batch:
self._reqs_to_process.add(req_id)
for req_id in metadata.reqs_not_processed:
self._reqs_to_process.discard(req_id)
assert req_id not in self._reqs_to_send
# Rebase scheduler-clock deadlines onto this worker's clock — see the
# equivalent block in pull_worker.start_load_kv for the rationale.
now_local = time.perf_counter()
for req_id, expiration_time in metadata.reqs_to_send.items():
if req_id in self._reqs_to_process:
if metadata.scheduler_clock:
expiration_time = now_local + (
expiration_time - metadata.scheduler_clock
)
self._reqs_to_send[req_id] = expiration_time
# Heartbeats still leave from the main thread (base worker behaviour).
self._send_heartbeats(metadata)
# --- Writer thread ------------------------------------------------- #
def _push_writer_loop(self) -> None:
sleep_s = _PUSH_WRITER_POLL_INTERVAL_MS / 1000.0
while not self._push_writer_stop.is_set():
try:
# 1. D registrations to send.
while True:
try:
rid, rd = self._reg_send_inbox.get_nowait()
except queue.Empty:
break
self._send_registration_to_p(rid, rd)
# 2. Deferred P→D pushes whose handshake just completed; do xfer now
while True:
try:
rid, blocks, rd = self._deferred_push_inbox.get_nowait()
except queue.Empty:
break
self._do_start_push_kv(rid, blocks, rd)
# 3. P-side finished blocks; match against pending regs.
while True:
try:
rid, blocks = self._finished_blocks_inbox.get_nowait()
except queue.Empty:
break
matched = self._pop_matching_registration(rid)
if matched is not None:
self._do_start_push_kv(rid, blocks, matched)
else:
self._push_finished_blocks[rid] = blocks
# 3b. Evict finished blocks for requests that have either
# completed (WRITE acknowledged) or whose lease expired
# without a D registration. Drop pending registrations
# for the same reason so we don't leak state.
while True:
try:
rid = self._evict_finished_inbox.get_nowait()
except queue.Empty:
break
self._push_finished_blocks.pop(rid, None)
self._pending_d_registrations.pop(rid, None)
# 4. NIXL notifs: route PUSH_REG; forward the rest.
for notifs in self.nixl_wrapper.get_new_notifs().values():
for notif in notifs:
if notif.startswith(PUSH_REG_NOTIF_PREFIX):
self._handle_push_reg_notif(notif)
else:
self._pending_completion_notifs.put(notif)
except Exception:
logger.exception("nixl-push-writer error; continuing")
# Self-poll only while there is no other wake source: P-side
# finished blocks waiting for a D PUSH_REG match. All other
# progress is event-driven (see module docstring).
if self._push_finished_blocks:
self._push_writer_stop.wait(timeout=sleep_s)
else:
self._push_writer_wake.wait()
self._push_writer_wake.clear()
def _handle_push_reg_notif(self, notif: bytes) -> None:
try:
reg_data = msgspec.msgpack.decode(notif[len(PUSH_REG_NOTIF_PREFIX) :])
except Exception:
logger.exception("Failed to decode PUSH_REG notification payload")
return
rid = reg_data.get("request_id") if isinstance(reg_data, dict) else None
if not isinstance(rid, str):
logger.warning("PUSH_REG notif missing request_id; dropping")
return
match = self._pop_matching_finished_blocks(rid)
if match is not None:
fin_id, blocks = match
self._do_start_push_kv(fin_id, blocks, reg_data)
else:
self._pending_d_registrations[rid] = reg_data
# --- D-side registration send (writer thread) ---------------------- #
def _send_registration_to_p(
self,
req_id: str,
reg_data: dict[str, Any],
) -> None:
"""Handshake (if needed) then send PUSH_REG. ``send_notif`` always
executes on the writer; the handshake runs on the background executor
and the request is re-queued onto ``_reg_send_inbox`` once it
completes (at which point ``_ensure_handshake`` returns ``None`` and we
send directly)."""
remote_pp_size = reg_data.get("remote_pp_size", 1)
fut = self._ensure_handshake(
reg_data["remote_engine_id"],
reg_data["remote_host"],
reg_data["remote_port"],
reg_data["remote_tp_size"],
pp_size=remote_pp_size,
# D only ever sends PUSH_REG notifs to P and never reads or writes
# P's memory in push mode, so it never needs the transfer
# descriptors set up by the full add_remote_agent path.
notif_agents_only=True,
)
if fut is None:
self._do_send_reg_notif(req_id, reg_data)
return
def _on_handshake(
f: Future[tuple[dict[tuple[int, int], str], float]],
rid: str = req_id,
rd: dict[str, Any] = reg_data,
) -> None:
try:
f.result()
except Exception as e:
self._log_failure(
failure_type="push_reg_handshake_failed", req_id=rid, error=e
)
self._failed_recv_reqs.put(rid)
return
# Re-queue for the writer to send now that the handshake is done.
self._reg_send_inbox.put((rid, rd))
# Wake the writer so it sends the PUSH_REG promptly even if
# otherwise parked.
self._push_writer_wake.set()
fut.add_done_callback(_on_handshake)
def _do_send_reg_notif(self, req_id: str, reg_data: dict[str, Any]) -> None:
engine_id = reg_data["remote_engine_id"]
notif_msg = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(reg_data)
# _remote_agents is mutated on other threads; snapshot under the lock.
with self._handshake_lock:
agents = dict(self._remote_agents.get(engine_id) or {})
if not agents:
logger.error(
"No remote agents for engine %s; cannot send registration for %s",
engine_id,
req_id,
)
self.xfer_stats.record_failed_notification()
self._failed_recv_reqs.put(req_id)
return
for rank, agent_name in agents.items():
try:
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_msg)
except Exception as e:
self._log_failure(
failure_type="push_reg_notif_failed",
req_id=req_id,
error=e,
remote_rank=rank,
)
self.xfer_stats.record_failed_notification()
# Earlier registrations may still trigger WRITEs into D's blocks.
# Keep the receive pending until those writes are finished.
logger.debug(
"Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg)
)
# --- Matching helpers --------------------------------------------- #
def _pop_matching_registration(self, request_id: str) -> dict[str, Any] | None:
"""Pop the D-side registration matching *request_id*.
Exact key first, then a match after stripping the random suffix from
both sides. No match leaves the request unmatched (push not started).
"""
data = self._pending_d_registrations.pop(request_id, None)
if data is not None:
return data
base_id = get_base_request_id(request_id)
for reg_id in list(self._pending_d_registrations):
if get_base_request_id(reg_id) == base_id:
return self._pending_d_registrations.pop(reg_id)
return None
def _pop_matching_finished_blocks(
self, request_id: str
) -> tuple[str, BlockIds] | None:
"""Pop the P-side finished blocks matching *request_id*.
Same lookup as ``_pop_matching_registration``: exact key, then a
match after stripping the random suffix from both sides.
"""
blocks = self._push_finished_blocks.pop(request_id, None)
if blocks is not None:
return request_id, blocks
base_id = get_base_request_id(request_id)
for fin_id in list(self._push_finished_blocks):
if get_base_request_id(fin_id) == base_id:
return fin_id, self._push_finished_blocks.pop(fin_id)
return None
# --- WRITE transfer logic (writer thread) ------------------------- #
def _do_start_push_kv(
self,
request_id: str,
local_block_ids: BlockIds,
registration_data: dict[str, Any],
) -> None:
"""Start push-based KV transfer from P worker to D node.
The P→D handshake runs on the base worker's background executor.
If it isn't ready yet we register a completion callback, defer the
WRITE, and re-drive this request via ``_deferred_push_inbox`` once
the handshake resolves -- so the writer thread never blocks on the
network (mirrors ``_send_registration_to_p``).
"""
if not local_block_ids:
logger.warning("No local blocks to push for request %s", request_id)
return
# ``local_block_ids`` are P's logical block IDs; ``remote_block_ids``
# (D's, from the PUSH_REG notif) are also logical.
decode_engine_id = registration_data["decode_engine_id"]
remote_block_ids = registration_data["local_block_ids"]
decode_request_id = registration_data["request_id"]
# Runs on the background executor; defer the WRITE until it's ready.
fut = self._ensure_handshake(
decode_engine_id,
registration_data["decode_host"],
registration_data["decode_port"],
registration_data["decode_tp_size"],
)
if fut is not None:
def _on_handshake(
f: Future[tuple[dict[tuple[int, int], str], float]],
rid: str = request_id,
blocks: BlockIds = local_block_ids,
rd: dict[str, Any] = registration_data,
) -> None:
if (e := f.exception()) is not None:
# The engine reclaims the blocks via the TTL so we dont free here
self._log_failure(
failure_type="push_handshake_failed", req_id=rid, error=e
)
return
self._deferred_push_inbox.put((rid, blocks, rd))
self._push_writer_wake.set()
fut.add_done_callback(_on_handshake)
return
# Keep the engine alive while it is actively receiving pushes, mirroring
# how pull-mode transfers touch _engine_last_active in start_load_kv.
self._engine_last_active[decode_engine_id] = time.perf_counter()
logical_local = self._as_grouped_block_ids(local_block_ids)
logical_remote = self._as_grouped_block_ids(remote_block_ids)
physical_local = self._logical_to_kernel_block_ids(
logical_local, self._physical_blocks_per_logical_kv_block
)
push_meta = ReqMeta(
local_block_ids=logical_local,
local_physical_block_ids=physical_local,
tp_size=self.world_size,
remote=RemoteMeta(
block_ids=logical_remote,
host="",
port=0,
engine_id=decode_engine_id,
request_id=decode_request_id,
),
)
t0 = time.perf_counter()
self._xfer_blocks_for_req(req_id=request_id, meta=push_meta)
elapsed_ms = (time.perf_counter() - t0) * 1000.0
if elapsed_ms > 200.0:
logger.warning(
"_do_start_push_kv for %s took %.1fms (slow NIXL submission)",
request_id,
elapsed_ms,
)
@staticmethod
def _as_grouped_block_ids(block_ids: BlockIds) -> BlockIds:
"""Normalise a sequence of block IDs to a tuple-of-groups shape.
``BlockIds`` is canonically a tuple of per-group lists, but some
registration payloads collapse a single-group case to a flat
list. Re-wrap that case so downstream group-aware helpers see a
consistent shape."""
if block_ids and not isinstance(block_ids[0], (list, tuple)):
return (list(block_ids),)
return block_ids
def _xfer_blocks_for_req(self, req_id: str, meta: ReqMeta):
"""Issue WRITE transfers to one or more remote TP ranks."""
assert meta.remote is not None and self.transfer_topo is not None
engine_id = meta.remote.engine_id
plan = self.tp_mappings[engine_id]
remote_info = self.transfer_topo.get_engine_info(engine_id)
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
# Expand D's logical IDs using the ratio learned during the
# NIXL handshake. ``meta`` is freshly built by
# ``_do_start_push_kv`` so mutating it here is safe.
meta.remote.block_ids = self._logical_to_kernel_block_ids(
meta.remote.block_ids,
remote_info.remote_physical_blocks_per_logical,
)
remote_block_ids = meta.remote.block_ids
local_block_ids = meta.local_physical_block_ids
local_region_groups = self.region_group_ids
remote_region_groups = self.dst_region_group_ids[engine_id]
groups_differ = local_region_groups != remote_region_groups
if groups_differ and not self._transfer_layer_group_ids:
raise NotImplementedError(
"NixlPushConnector does not support different producer and "
"consumer cache-group layouts"
)
# MLA latent is replicated across D's TP ranks: the tp-mapping
# collapses it to one rank (fine for reads), but push must WRITE every
# D rank or the rest decode stale KV. For hybrid MLA+SSM the sharded
# SSM state already targets every covered D rank, so only the
# attention groups need widening; pure MLA writes to all handshaked
# ranks (only the dst differs per rank).
replicate_attn = self.use_mla and tp_ratio < 0
if replicate_attn and not self._has_mamba:
assert len(plan.all_source_ranks) == 1
write_ranks = sorted(self.dst_xfer_side_handles[engine_id])
else:
write_ranks = list(plan.all_source_ranks)
num_groups = len(local_block_ids)
def group_ids(block_ids: BlockIds, rank: int) -> BlockIds:
return [
list(block_ids[g])
if (self._is_csa_linear and tp_ratio < 0)
or (replicate_attn and _is_attention_spec(self._group_spec_types[g]))
or rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
]
read_specs = [
ReadSpec(
remote_rank=rank,
local_block_ids=group_ids(local_block_ids, rank),
remote_block_ids=group_ids(remote_block_ids, rank),
)
for rank in write_ranks
]
handles: list[int] = []
for i, spec in enumerate(read_specs):
remote_block_size = remote_info.remote_block_size
logger.debug(
"Remote agent %s available, calling _xfer_blocks"
" on remote rank %s with remote block size %s for req %s",
meta.remote.engine_id,
spec.remote_rank,
remote_block_size,
req_id,
)
if tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1):
# Multiple targets: write each rank its chunk of local memory.
# Hybrid MLA+SSM also lands here: its split handles replicate
# the attention descriptors and chunk only the SSM state.
split_key = (tp_ratio, remote_block_size)
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
else:
local_xfer_side_handle = self.src_xfer_handles_by_block_size[
remote_block_size
]
remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][
spec.remote_rank
]
handle = self._xfer_blocks(
read_spec=spec,
request_id=req_id,
dst_engine_id=meta.remote.engine_id,
remote_request_id=meta.remote.request_id,
local_xfer_side_handle=local_xfer_side_handle,
remote_xfer_side_handle=remote_xfer_side_handle,
)
if handle is not None:
handles.append(handle)
# Publish all the request's WRITE handles in one locked update: a
# partial set would let ``_pop_done_transfers`` finish the request
# early, then double-report it as the remaining writes land.
if handles:
with self._sending_transfers_lock:
self._sending_transfers[req_id].extend(handles)
def _xfer_blocks(
self,
read_spec: ReadSpec,
dst_engine_id: str,
request_id: str,
remote_request_id: str,
local_xfer_side_handle: int,
remote_xfer_side_handle: int,
) -> int | None:
"""Post a WRITE point-to-point xfer request.
Returns the in-flight transfer handle (so the caller can track all of
a request's handles atomically), or ``None`` if nothing was submitted.
"""
assert self.transfer_topo is not None
remote_rank = read_spec.remote_rank
local_block_ids = read_spec.local_block_ids
remote_block_ids = read_spec.remote_block_ids
remote_info = self.transfer_topo.get_engine_info(dst_engine_id)
block_size_ratio = self.transfer_topo.block_size_ratio(
remote_info.remote_block_size
)
if block_size_ratio > 1:
local_block_ids, remote_block_ids = (
self._map_block_ids_for_block_size_ratio(
local_block_ids, remote_block_ids, block_size_ratio
)
)
notif_id = f"{remote_request_id}:{self.world_size}".encode()
if len(local_block_ids) == 0:
logger.warning("No blocks to push for request %s", request_id)
return None
# Prefix caching: D allocated only uncached blocks, so on a partial hit it
# sends fewer than P's. End-trim P's blocks to that same suffix so we WRITE only
# the uncomputed tail into D's slots. Runs on kernel ids, post-expansion.
remote_block_ids, local_block_ids = self._apply_prefix_caching(
decode_block_ids=remote_block_ids,
prefill_block_ids=local_block_ids,
decode_physical_per_logical=remote_info.remote_physical_blocks_per_logical,
prefill_physical_per_logical=self._physical_blocks_per_logical_kv_block,
)
local_block_ids = list(local_block_ids)
remote_block_ids = list(remote_block_ids)
assert len(local_block_ids) == len(remote_block_ids), (
f"push group-count mismatch for {request_id}: {len(local_block_ids)} "
f"local vs {len(remote_block_ids)} remote groups"
)
for i in range(len(local_block_ids)):
assert len(local_block_ids[i]) == len(remote_block_ids[i]), (
f"push block-count mismatch for {request_id} group {i}: "
f"{len(local_block_ids[i])} local vs "
f"{len(remote_block_ids[i])} remote blocks"
)
# Get descs ids.
remote_block_descs_ids = self._compute_desc_ids(
block_ids=remote_block_ids,
dst_num_blocks=self.dst_num_blocks[dst_engine_id],
block_size_ratio=None,
physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical,
region_num_blocks=self.dst_region_num_blocks[dst_engine_id],
region_group_ids=self.dst_region_group_ids[dst_engine_id],
uses_region_group_mapping=self.dst_uses_region_group_mapping[dst_engine_id],
)
local_block_descs_ids = self._compute_desc_ids(
block_ids=local_block_ids,
dst_num_blocks=self.dst_num_blocks[self.engine_id],
block_size_ratio=block_size_ratio,
physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block,
region_num_blocks=self.dst_region_num_blocks[self.engine_id],
region_group_ids=self.region_group_ids,
uses_region_group_mapping=self._uses_region_group_mapping,
)
assert len(local_block_descs_ids) == len(remote_block_descs_ids)
handle = None
try:
handle = self.nixl_wrapper.make_prepped_xfer(
"WRITE",
local_xfer_side_handle,
local_block_descs_ids,
remote_xfer_side_handle,
remote_block_descs_ids,
notif_msg=notif_id,
)
self.nixl_wrapper.transfer(handle)
# Caller tracks the handle (atomically with the request's other
# writes) so P can free blocks once all of them are done.
return handle
except Exception as e:
self._log_failure(
failure_type="transfer_setup_failed",
req_id=request_id,
msg="Push WRITE submission failed; releasing handle",
error=e,
dst_engine_id=dst_engine_id,
remote_rank=remote_rank,
)
# On the P side this WRITE failure is purely outbound; we
# don't have a ``_recving_metadata`` entry to invalidate, so
# we just release the handle and let the engine reschedule
# via the lease / watchdog.
with self._sending_transfers_lock:
self._send_failures.add(request_id)
if not self._handle_failed_transfer(request_id, handle):
return handle
return None
# --- Notification handling on engine main thread ------------------ #
def _get_new_notifs(self) -> set[str]:
"""Drain HB / completion notifs forwarded by the writer thread.
The writer owns ``nixl_wrapper.get_new_notifs`` for push; PUSH_REG
notifs are handled there. P learns of finished pushes from its WRITE
handles, so this never reports finished sends.
"""
while True:
try:
notif = self._pending_completion_notifs.get_nowait()
except queue.Empty:
break
msg = notif.decode("utf-8")
if msg.startswith("HB:"):
self._handle_heartbeat(msg[3:])
continue
req_id, tp_size = msg.rsplit(":", 1)
if (meta := self._recving_metadata.get(req_id)) is None:
# The lease may have expired before the notif arrived.
self.xfer_stats.record_notification_after_expiry()
logger.error(
"Unrecognized request %s notif (may have expired).", req_id
)
continue
# Consumer waits for one notif per producer rank writing
# here: pp_size stages * producers-per-consumer (>1 when
# producer TP > consumer TP; tp_size is the producer TP).
producers_per_consumer = max(1, int(tp_size) // self.world_size)
expected_notifs = meta.pp_size * producers_per_consumer
self.consumer_notification_counts_by_req[req_id] += 1
if self.consumer_notification_counts_by_req[req_id] < expected_notifs:
continue
del self.consumer_notification_counts_by_req[req_id]
# P drove the transfer (we own no NIXL handle), so
# materialise an empty ``_recving_transfers`` entry for
# ``_pop_done_transfers`` to report done.
self._recving_transfers.setdefault(req_id, [])
return set()
def get_transfer_results(self) -> KVConnectorTransferResults:
# Engine main thread asking for completions: also wake the writer
# so it gets a chance to drain NIXL notifs (heartbeats, completion
# notifs, late PUSH_REGs) even if it had been parked.
self._push_writer_wake.set()
results = super().get_transfer_results()
done_sending = results.finished_sending
# ``_pop_done_transfers`` mutates ``_sending_transfers``; the
# writer thread also appends to it, so guard the pop.
with self._sending_transfers_lock:
done_pushing, failed_pushing = self._pop_done_transfers(
self._sending_transfers
)
# Remember failures until the final sibling WRITE completes.
self._send_failures.update(failed_pushing)
successful = {
req_id
for req_id in done_pushing - self._send_failures
if req_id in self._reqs_to_send or req_id in self._reqs_to_process
}
self._send_failures.difference_update(done_pushing | done_sending)
# Expired requests were already reported, even if their WRITEs finish later.
done_pushing = successful
for req_id in done_pushing:
self._reqs_to_send.pop(req_id, None)
self._reqs_to_process.discard(req_id)
done_sending.add(req_id)
# Tell the writer to drop any state it still holds for any
# request that just finished (push completed) or expired
# (lease ran out without a D registration ever arriving).
for req_id in done_sending:
self._evict_finished_inbox.put(req_id)
if done_sending:
self._push_writer_wake.set()
return results
@@ -1,12 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backward-compatible re-export of NixlPullConnectorScheduler."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
# Backward compatibility: NixlConnectorScheduler is the pull-based scheduler.
NixlConnectorScheduler = NixlPullConnectorScheduler
__all__ = ["NixlConnectorScheduler", "NixlPullConnectorScheduler"]
@@ -1,328 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stats and Prometheus metrics for the NIXL connector."""
import copy
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import numpy as np
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.v1.metrics.utils import create_metric_per_engine
if TYPE_CHECKING:
from vllm.distributed.nixl_utils import nixlXferTelemetry
@dataclass
class NixlKVConnectorStats(KVConnectorStats):
"""Container for transfer performance metrics."""
def __post_init__(self):
if not self.data:
# Empty container init, no data is passed in.
self.reset()
def reset(self):
# Must be serializable
self.data: dict[str, list[float | int]] = {
"transfer_duration": [],
"post_duration": [],
"bytes_transferred": [],
"num_descriptors": [],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
"num_notifications_after_expiry": [],
}
def record_transfer(self, res: "nixlXferTelemetry | None"):
# Keep metrics units consistent with rest of the code: time us->s
if res is None:
return
self.data["transfer_duration"].append(res.xferDuration / 1e6)
self.data["post_duration"].append(res.postDuration / 1e6)
self.data["bytes_transferred"].append(res.totalBytes)
self.data["num_descriptors"].append(res.descCount)
def record_failed_transfer(self):
"""Record a failed NIXL transfer operation."""
self.data["num_failed_transfers"].append(1)
def record_failed_notification(self):
"""Record a failed NIXL notification (send_notif)."""
self.data["num_failed_notifications"].append(1)
def record_failed_handshake(self):
"""Record a failed NIXL handshake."""
self.data["num_failed_handshakes"].append(1)
def record_kv_expired_req(self):
"""Record a request that had its KV blocks expire."""
self.data["num_kv_expired_reqs"].append(1)
def record_notification_after_expiry(self):
"""Record a completion notification that arrived after its request expired."""
self.data["num_notifications_after_expiry"].append(1)
def clone_and_reset(self) -> "NixlKVConnectorStats":
old = copy.copy(self)
self.reset()
return old
def is_empty(self) -> bool:
# Do not discard metrics update that are entirely failures related.
return (
self.num_successful_transfers == 0
and len(self.data["num_failed_transfers"]) == 0
and len(self.data["num_failed_notifications"]) == 0
and len(self.data["num_failed_handshakes"]) == 0
and len(self.data["num_kv_expired_reqs"]) == 0
and len(self.data["num_notifications_after_expiry"]) == 0
)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
accumulator = self.data[k]
assert isinstance(accumulator, list)
accumulator.extend(v)
return self
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging.
# Failure counts are reported on every interval: transfer, handshake
# and notification failures are grouped as sporadic
# lower-transport-layer events, while KV expiry is reported separately
# as it is an actionable autoscaler signal rather than a transport
# issue. Notifications arriving after expiry are surfaced too, as the
# reader may have consumed KV blocks that were already reused.
failure_counts = {
"Num failed transfers": len(self.data["num_failed_transfers"])
+ len(self.data["num_failed_handshakes"])
+ len(self.data["num_failed_notifications"]),
"Num KV expired reqs": len(self.data["num_kv_expired_reqs"]),
"Num notifs after expiry": len(self.data["num_notifications_after_expiry"]),
}
if self.num_successful_transfers == 0:
# Timing / throughput stats only cover successful transfers. If
# all requests in the interval were unsuccessful, the failure
# counts above still surface what happened.
return {
"Num successful transfers": 0,
"Avg xfer time (ms)": 0,
"P90 xfer time (ms)": 0,
"Avg post time (ms)": 0,
"P90 post time (ms)": 0,
"Avg MB per transfer": 0,
"Throughput (MB/s)": 0,
"Avg number of descriptors": 0,
**failure_counts,
}
xfer_time = np.asarray(self.data["transfer_duration"])
post_time = np.asarray(self.data["post_duration"])
# Convert to MB for CLI logging.
mb = np.asarray(self.data["bytes_transferred"]) / 2**20
descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32)
n = len(descs)
assert n == self.num_successful_transfers
total_mb = mb.sum()
avg_mb = total_mb / n
total_time_seconds = xfer_time.sum()
throughput_mb_s = total_mb / total_time_seconds
return {
"Num successful transfers": n,
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
"Avg post time (ms)": round(post_time.mean().item() * 1e3, 3),
"P90 post time (ms)": round(np.percentile(post_time, 90).item() * 1e3, 3),
"Avg MB per transfer": round(avg_mb.item(), 3),
"Throughput (MB/s)": round(throughput_mb_s.item(), 3),
"Avg number of descriptors": round(descs.mean().item(), 1),
**failure_counts,
}
@property
def num_successful_transfers(self) -> int:
return len(self.data["transfer_duration"])
class NixlPromMetrics(KVConnectorPromMetrics):
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
buckets = [
0.001,
0.005,
0.01,
0.025,
0.05,
0.075,
0.1,
0.2,
0.3,
0.5,
0.75,
1.0,
5.0,
]
nixl_histogram_xfer_time = self._histogram_cls(
name="vllm:nixl_xfer_time_seconds",
documentation="Histogram of transfer duration for NIXL KV Cache transfers.",
buckets=buckets[1:],
labelnames=labelnames,
)
self.nixl_histogram_xfer_time = create_metric_per_engine(
nixl_histogram_xfer_time, self.per_engine_labelvalues
)
nixl_histogram_post_time = self._histogram_cls(
name="vllm:nixl_post_time_seconds",
documentation="Histogram of transfer post time for NIXL KV"
" Cache transfers.",
buckets=buckets,
labelnames=labelnames,
)
self.nixl_histogram_post_time = create_metric_per_engine(
nixl_histogram_post_time, self.per_engine_labelvalues
)
# uniform 2kb to 16gb range
buckets = [2 ** (10 + i) for i in range(1, 25, 2)]
nixl_histogram_bytes_transferred = self._histogram_cls(
name="vllm:nixl_bytes_transferred",
documentation="Histogram of bytes transferred per NIXL KV Cache transfers.",
buckets=buckets,
labelnames=labelnames,
)
self.nixl_histogram_bytes_transferred = create_metric_per_engine(
nixl_histogram_bytes_transferred, self.per_engine_labelvalues
)
buckets = [
10,
20,
30,
50,
75,
100,
200,
400,
1000,
2000,
4000,
10000,
20000,
50000,
]
nixl_histogram_num_descriptors = self._histogram_cls(
name="vllm:nixl_num_descriptors",
documentation="Histogram of number of descriptors per NIXL"
" KV Cache transfers.",
buckets=buckets,
labelnames=labelnames,
)
self.nixl_histogram_num_descriptors = create_metric_per_engine(
nixl_histogram_num_descriptors, self.per_engine_labelvalues
)
counter_nixl_num_failed_transfers = self._counter_cls(
name="vllm:nixl_num_failed_transfers",
documentation="Number of failed NIXL KV Cache transfers, including"
" handshake and notification failures. NOTE: KV expiry is tracked"
" separately in vllm:nixl_num_kv_expired_reqs.",
labelnames=labelnames,
)
self.counter_nixl_num_failed_transfers = create_metric_per_engine(
counter_nixl_num_failed_transfers, self.per_engine_labelvalues
)
counter_nixl_num_failed_notifications = self._counter_cls(
name="vllm:nixl_num_failed_notifications",
documentation="Number of failed NIXL KV Cache notifications. "
"Retained for compatibility; these failures are also included in "
"vllm:nixl_num_failed_transfers.",
labelnames=labelnames,
)
self.counter_nixl_num_failed_notifications = create_metric_per_engine(
counter_nixl_num_failed_notifications, self.per_engine_labelvalues
)
counter_nixl_num_kv_expired_reqs = self._counter_cls(
name="vllm:nixl_num_kv_expired_reqs",
documentation="Number of requests that had their KV expire. "
"NOTE: This metric is tracked on the P instance.",
labelnames=labelnames,
)
self.counter_nixl_num_kv_expired_reqs = create_metric_per_engine(
counter_nixl_num_kv_expired_reqs, self.per_engine_labelvalues
)
counter_nixl_num_notifications_after_expiry = self._counter_cls(
name="vllm:nixl_num_notifications_after_expiry",
documentation="Number of completion notifications for requests that "
"were no longer tracked, usually because their KV lease expired. The "
"KV blocks may have been reused before the transfer finished. Counted "
"per notification (one per remote rank), not per request.",
labelnames=labelnames,
)
self.counter_nixl_num_notifications_after_expiry = create_metric_per_engine(
counter_nixl_num_notifications_after_expiry, self.per_engine_labelvalues
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
for prom_obj, list_item_key in zip(
[
self.nixl_histogram_xfer_time,
self.nixl_histogram_post_time,
self.nixl_histogram_bytes_transferred,
self.nixl_histogram_num_descriptors,
],
[
"transfer_duration",
"post_duration",
"bytes_transferred",
"num_descriptors",
],
):
for list_item in transfer_stats_data[list_item_key]:
prom_obj[engine_idx].observe(list_item)
for counter_obj, counter_item_keys in zip(
[
self.counter_nixl_num_failed_transfers,
self.counter_nixl_num_failed_notifications,
self.counter_nixl_num_kv_expired_reqs,
self.counter_nixl_num_notifications_after_expiry,
],
[
# Transfer, handshake and notification failures are grouped:
# all are sporadic lower-transport-layer events. KV expiry is
# reported separately as it signals autoscaler behavior, not
# transport health.
(
"num_failed_transfers",
"num_failed_handshakes",
"num_failed_notifications",
),
("num_failed_notifications",),
("num_kv_expired_reqs",),
("num_notifications_after_expiry",),
],
):
for counter_item_key in counter_item_keys:
for list_item in transfer_stats_data[counter_item_key]:
counter_obj[engine_idx].inc(list_item)
@@ -1,161 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TP mapping computation for NIXL KV cache transfers."""
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
TransferTopology,
)
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, MambaSpec
# ======================================================================
# Data structures
# ======================================================================
@dataclass(frozen=True)
class ReadSpec:
"""Specification for a single remote block read operation."""
remote_rank: int
local_block_ids: BlockIds
remote_block_ids: BlockIds
block_ids_by_region: bool = False
def _is_attention_spec(spec_type: type[KVCacheSpec]) -> bool:
return issubclass(spec_type, AttentionSpec)
def _is_ssm_spec(spec_type: type[KVCacheSpec]) -> bool:
return issubclass(spec_type, MambaSpec)
@dataclass(frozen=True)
class TPMapping:
"""Complete local-to-remote TP mapping for one remote engine.
Generated once per remote engine during handshake.
"""
# Remote TP ranks that this local rank reads from, per group.
# Position = local piece index.
source_ranks_per_group: tuple[tuple[int, ...], ...]
# Superset of all source ranks (union of all groups).
all_source_ranks: tuple[int, ...]
# Maps each source rank to its FA head slot index.
rank_to_attention_slot: dict[int, int]
# FA head offset factor for hetero-TP (D_TP > P_TP).
rank_offset_factor: int
# Local ranks (in aggregate) that read from a given source rank. The producer frees
# a request's blocks only once that many notifications have come in.
local_consumers: int = 1
# ======================================================================
# TP mapping computation
# ======================================================================
def compute_tp_mapping(
transfer_topology: TransferTopology,
remote_tp_size: int,
group_spec_types: tuple[type[KVCacheSpec], ...],
remote_dcp_size: int = 1,
) -> TPMapping:
"""Build the complete local-to-remote TP mapping.
Computes source ranks, head slot assignments, and the rank offset
factor in a single pass.
DCP support is scoped to MLA only, with a side is either fully replicated or fully
sharded. DCP-branch reuses the same rank set used at handshake selection.
"""
tp_rank = transfer_topology.tp_rank
tp_size = transfer_topology.tp_size
total_num_kv_heads = transfer_topology.total_num_kv_heads
# --- Attention source ranks ---
if transfer_topology.is_mla or tp_size >= remote_tp_size:
if transfer_topology.is_mla and remote_dcp_size > 1:
attn_ranks = transfer_topology.dcp_source_ranks(
remote_tp_size, remote_dcp_size
)
else:
# D (local TP) > P (remote TP): multiple local ranks read different chunks
# from *one* remote rank, corresponding to different kv heads.
# For MLA, we only need one remote since cache is duplicated. When
# P TP=k*TP k, this will spread mla ranks to read from remote k*tp_rank.
attn_ranks = [tp_rank * remote_tp_size // tp_size]
else:
# P (remote TP) > D (local TP): one local rank
# reads from multiple remote ranks.
# GQA dedup: when K < remote_tp_size, several remote ranks
# hold the same KV head. np.unique keeps only the first
# rank per unique head so we don't issue redundant reads.
abs_tp = remote_tp_size // tp_size
start = tp_rank * abs_tp
heads = np.arange(start, start + abs_tp) * total_num_kv_heads // remote_tp_size
_, unique_idx = np.unique(heads, return_index=True)
attn_ranks = (start + np.sort(unique_idx)).tolist()
# --- SSM source ranks ---
has_ssm = any(_is_ssm_spec(t) for t in group_spec_types)
if has_ssm:
if tp_size < remote_tp_size:
abs_tp = remote_tp_size // tp_size
ssm_ranks = list(range(tp_rank * abs_tp, (tp_rank + 1) * abs_tp))
else:
ssm_ranks = list(attn_ranks)
else:
ssm_ranks = []
all_ranks = sorted(set(attn_ranks) | set(ssm_ranks))
# --- Per-group ordered source ranks ---
source_ranks_per_group = tuple(
tuple(ssm_ranks) if _is_ssm_spec(t) else tuple(attn_ranks)
for t in group_spec_types
)
# --- Attention head slots ---
head_to_slot: dict[int, int] = {}
for i, r in enumerate(attn_ranks):
head_to_slot[r * total_num_kv_heads // remote_tp_size] = i
rank_to_attention_slot = {
r: head_to_slot.get(r * total_num_kv_heads // remote_tp_size, 0)
for r in all_ranks
}
# --- Rank offset factor ---
if transfer_topology.is_mla or tp_size <= remote_tp_size:
# We don't index into remote for reading, no offset needed.
rank_offset_factor = 0
elif tp_size > total_num_kv_heads:
local_head = tp_rank * total_num_kv_heads // tp_size
p_start = attn_ranks[0] * total_num_kv_heads // remote_tp_size
rank_offset_factor = local_head - p_start
else:
# D TP > P TP: we index into remote to read different heads depending on rank.
rank_offset_factor = tp_rank % (tp_size // remote_tp_size)
local_consumers = transfer_topology.dcp_consumer_count(
remote_tp_size, remote_dcp_size
)
return TPMapping(
source_ranks_per_group=source_ranks_per_group,
all_source_ranks=tuple(all_ranks),
rank_to_attention_slot=rank_to_attention_slot,
rank_offset_factor=rank_offset_factor,
local_consumers=local_consumers,
)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared constants, lazy imports and helpers for the NIXL connector."""
import contextlib
from collections.abc import Iterator
from typing import Any
import regex as re
import zmq
from vllm.platforms import current_platform
from vllm.utils.network_utils import make_zmq_socket
from vllm.v1.kv_cache_interface import KVCacheSpec, UniformTypeKVCacheSpecs
# Supported platforms and types of kv transfer buffer.
# {device: tuple of supported kv buffer types}
_NIXL_SUPPORTED_DEVICE = {
"cuda": (
"cuda",
"cpu",
),
"tpu": ("cpu",),
"xpu": (
"cpu",
"xpu",
),
"cpu": ("cpu",),
}
# support for oot platform by providing mapping in current_platform
_NIXL_SUPPORTED_DEVICE.update(current_platform.get_nixl_supported_devices())
# TODO: merge with vllm.utils.network_utils.zmq_socket_ctx
@contextlib.contextmanager
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
"""Context manager for a ZMQ socket."""
if socket_type not in (zmq.ROUTER, zmq.REQ):
raise ValueError(f"Unexpected socket type: {socket_type}")
ctx: zmq.Context | None = None
try:
ctx = zmq.Context() # type: ignore[attr-defined]
yield make_zmq_socket(
ctx=ctx, path=addr, socket_type=socket_type, bind=socket_type == zmq.ROUTER
)
finally:
if ctx is not None:
ctx.destroy(linger=0)
def get_representative_spec_type(spec: KVCacheSpec) -> type[KVCacheSpec]:
if isinstance(spec, UniformTypeKVCacheSpecs):
# All inner specs are the same type; pick any.
inner = next(iter(spec.kv_cache_specs.values()))
return type(inner)
return type(spec)
# Trailing 8-hex randomization suffix appended by
# ``input_processor.assign_request_id`` as ``-{random_uuid():.8}``.
_RANDOM_SUFFIX_RE = re.compile(r"-[0-9a-f]{8}$", re.IGNORECASE)
def get_base_request_id(request_id: str) -> str:
"""Strip the per-request ``-<8 hex>`` randomization suffix, if present."""
return _RANDOM_SUFFIX_RE.sub("", request_id)
@@ -1,13 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backward-compatible re-export of NixlPullConnectorWorker."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
NixlPullConnectorWorker,
)
# Backward compatibility: NixlConnectorWorker is the pull-based worker.
NixlConnectorWorker = NixlPullConnectorWorker
__all__ = ["NixlConnectorWorker", "NixlPullConnectorWorker"]
@@ -1,386 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SimpleCPUOffloadConnector: minimal CPU KV cache offloading."""
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_events import KVCacheEvent
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.simple_kv_offload.manager import (
BoundaryStoreStats,
SimpleCPUOffloadScheduler,
)
from vllm.v1.simple_kv_offload.metadata import (
SimpleCPUOffloadMetadata,
)
from vllm.v1.simple_kv_offload.metrics import (
SimpleCPUOffloadPromMetrics,
SimpleCPUOffloadStats,
)
from vllm.v1.simple_kv_offload.worker import (
SimpleCPUOffloadWorker,
)
if TYPE_CHECKING:
from vllm.forward_context import ForwardContext
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
# Default CPU capacity: 8 GB
DEFAULT_CPU_CAPACITY_BYTES = 8 * (1024**3)
VALID_KV_OFFLOAD_BACKENDS = ("cpu", "disk")
# Keys that only apply to the disk backend, warned about under "cpu".
_DISK_ONLY_KEYS = (
"disk_path",
"disk_capacity_bytes",
"disk_buffer_slots",
"use_page_cache",
)
class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
"""CPU KV cache offloading with custom kernel transfers and BlockPool LRU."""
@property
def requires_kv_delivery(self) -> bool:
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
# future cache miss, so opt out of the producer-role default.
return False
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, role, kv_cache_config)
enable_prefix_caching = vllm_config.cache_config.enable_prefix_caching
extra_config = self._kv_transfer_config.kv_connector_extra_config or {}
cpu_capacity_bytes = int(
extra_config.get("cpu_bytes_to_use", DEFAULT_CPU_CAPACITY_BYTES)
)
# cpu_bytes_to_use is server-wide for compatibility;
# cpu_bytes_to_use_per_rank overrides for per-rank capacity.
world_size = vllm_config.parallel_config.world_size
cpu_capacity_per_rank = cpu_capacity_bytes // world_size
if "cpu_bytes_to_use_per_rank" in extra_config:
explicit = int(extra_config["cpu_bytes_to_use_per_rank"])
if explicit != cpu_capacity_per_rank:
logger.warning(
"cpu_bytes_to_use_per_rank (%.2f GB) != "
"cpu_bytes_to_use/world_size (%.2f GB). Using per-rank value.",
explicit / (1024**3),
cpu_capacity_per_rank / (1024**3),
)
cpu_capacity_per_rank = explicit
# When set, num_cpu_blocks overrides automatic pool sizing from byte budgets
# (including cpu_bytes_to_use and cpu_bytes_to_use_per_rank), allowing ranks
# with different KV layouts (e.g. PP stages) to use the same pool size.
# Byte budgets still bound capacity.
num_cpu_blocks = extra_config.get("num_cpu_blocks")
if num_cpu_blocks is not None and (
type(num_cpu_blocks) is not int or num_cpu_blocks <= 0
):
raise ValueError("num_cpu_blocks must be a positive integer")
lazy_offload = bool(extra_config.get("lazy_offload", False))
kv_offload_backend = str(extra_config.get("kv_offload_backend", "cpu"))
if kv_offload_backend not in VALID_KV_OFFLOAD_BACKENDS:
raise ValueError(
f"Unknown kv_offload_backend {kv_offload_backend!r}; "
f"expected one of {VALID_KV_OFFLOAD_BACKENDS}"
)
disk_mode = kv_offload_backend == "disk"
self._cache_hit_source = (
CacheHitSource.DISK if disk_mode else CacheHitSource.HOST
)
disk_path = extra_config.get("disk_path", None) or None
disk_capacity_bytes = int(
extra_config.get("disk_capacity_bytes", 100 * (1024**3))
)
disk_buffer_slots = max(1, int(extra_config.get("disk_buffer_slots", 2)))
use_page_cache = bool(extra_config.get("use_page_cache", False))
if disk_mode:
if disk_path is None:
raise ValueError(
'kv_offload_backend="disk" requires disk_path to be set.'
)
if disk_capacity_bytes <= 0:
raise ValueError(
'kv_offload_backend="disk" requires disk_capacity_bytes > 0.'
)
else:
ignored = [k for k in _DISK_ONLY_KEYS if k in extra_config]
if ignored:
logger.warning(
'kv_offload_backend is "cpu", ignoring disk-only config: %s. '
'Set kv_offload_backend="disk" to enable the disk backend.',
", ".join(ignored),
)
disk_path = None
self.scheduler_manager: SimpleCPUOffloadScheduler | None = None
self.worker_handler: SimpleCPUOffloadWorker | None = None
if not enable_prefix_caching:
logger.warning(
"Detected prefix caching disabled, disabling CPU offload "
"since it requires prefix caching."
)
return
logger.info(
"SimpleCPUOffloadConnector: role=%s, "
"per_rank=%.2f GB, world_size=%d, mode=%s, backend=%s, disk=%s",
role.name,
cpu_capacity_per_rank / (1024**3),
world_size,
"lazy" if lazy_offload else "eager",
kv_offload_backend,
disk_path or "none",
)
if num_cpu_blocks is not None:
logger.info(
"Using explicit num_cpu_blocks=%d for offload pool sizing "
"instead of automatic sizing from byte budgets.",
num_cpu_blocks,
)
if role == KVConnectorRole.SCHEDULER:
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
assert kv_cache_config is not None
scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self.scheduler_manager = SimpleCPUOffloadScheduler(
vllm_config,
kv_cache_config,
cpu_capacity_per_rank,
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
lazy_offload=lazy_offload,
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
use_page_cache=use_page_cache if disk_mode else False,
num_cpu_blocks=num_cpu_blocks,
)
elif role == KVConnectorRole.WORKER:
self.worker_handler = SimpleCPUOffloadWorker(
vllm_config,
kv_cache_config,
cpu_capacity_per_rank,
kv_offload_backend=kv_offload_backend,
disk_path=disk_path,
disk_capacity_bytes=disk_capacity_bytes,
disk_buffer_slots=disk_buffer_slots,
use_page_cache=use_page_cache,
num_cpu_blocks=num_cpu_blocks,
)
# --- Worker-side methods ---
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
if self.worker_handler is not None:
self.worker_handler.register_kv_caches(kv_caches)
def bind_connector_metadata(
self,
connector_metadata: KVConnectorMetadata,
) -> None:
super().bind_connector_metadata(connector_metadata)
if self.worker_handler is not None:
assert isinstance(connector_metadata, SimpleCPUOffloadMetadata)
self.worker_handler.bind_connector_metadata(connector_metadata)
def clear_connector_metadata(self) -> None:
super().clear_connector_metadata()
if self.worker_handler is not None:
self.worker_handler.clear_connector_metadata()
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata) -> None:
if self.worker_handler is not None:
assert isinstance(kv_connector_metadata, SimpleCPUOffloadMetadata)
self.worker_handler.handle_preemptions(kv_connector_metadata)
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
if self.worker_handler is not None:
self.worker_handler.start_load_kv()
def wait_for_layer_load(self, layer_name: str) -> None:
pass # Always load asynchronously, issued in start_load_kv()
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: "AttentionMetadata",
**kwargs: Any,
) -> None:
pass # Always save asynchronously, issued in wait_for_save()
def wait_for_save(self) -> None:
if self.worker_handler is not None:
self.worker_handler.wait_for_save()
def get_finished(
self,
finished_req_ids: set[str],
) -> tuple[set[str] | None, set[str] | None]:
if self.worker_handler is not None:
return self.worker_handler.get_finished(finished_req_ids)
return None, None
def build_connector_worker_meta(self):
if self.worker_handler is not None:
return self.worker_handler.build_connector_worker_meta()
return None
# --- Scheduler-side methods ---
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
if self.scheduler_manager is not None:
self.scheduler_manager.bind_gpu_block_pool(gpu_block_pool)
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
if self.scheduler_manager is not None:
return self.scheduler_manager.get_num_new_matched_tokens(
request, num_computed_tokens
)
return 0, False
def update_state_after_alloc(
self,
request: "Request",
blocks: "KVCacheBlocks",
num_external_tokens: int,
) -> None:
if self.scheduler_manager is not None:
self.scheduler_manager.update_state_after_alloc(
request, blocks, num_external_tokens
)
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
if self.scheduler_manager is not None:
return self.scheduler_manager.build_connector_meta(scheduler_output)
return SimpleCPUOffloadMetadata()
def update_connector_output(
self,
connector_output: KVConnectorOutput,
) -> None:
if self.scheduler_manager is not None:
self.scheduler_manager.update_connector_output(connector_output)
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
if self.scheduler_manager is not None:
return self.scheduler_manager.request_finished(request, block_ids)
return False, None
def request_finished_all_groups(
self,
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
if self.scheduler_manager is not None:
return self.scheduler_manager.request_finished_all_groups(
request, block_ids
)
return False, None
def has_pending_push_work(self) -> bool:
if self.scheduler_manager is not None:
return self.scheduler_manager.has_pending_stores()
return False
def get_boundary_store_stats(self) -> BoundaryStoreStats | None:
"""Return cumulative boundary-handoff store diagnostics."""
if self.scheduler_manager is not None:
return self.scheduler_manager.get_boundary_store_stats()
return None
def get_kv_connector_stats(self) -> KVConnectorStats | None:
# Worker-side callers may invoke this hook on connectors constructed
# via __new__ (as in test_worker.py), which lack scheduler_manager.
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
self, "scheduler_manager", None
)
if scheduler_manager is not None:
return scheduler_manager.get_stats()
return None
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
SimpleCPUOffloadStats(data=data)
if data is not None
else SimpleCPUOffloadStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return SimpleCPUOffloadPromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def take_events(self) -> Iterable[KVCacheEvent]:
if self.scheduler_manager is not None:
return self.scheduler_manager.take_events()
return []
# NOTE: Workers are not contacted. In-flight transfers drain naturally,
# and stale completions are ignored by the guarded
# SimpleCPUOffloadScheduler._process_store_event().
def reset_cache(self) -> bool | None:
if self.scheduler_manager is not None:
return self.scheduler_manager.reset()
return None
@@ -1,235 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mamba conv-state sub-projection decomposition for NIXL transfer.
With DS conv state layout (dim, state_len), sub-projections are
contiguous in memory. Each D rank reads its slices via separate
RDMA transfers — no P-side permutation needed.
Supported model types:
- Mamba1: conv = [x], temporal = (intermediate_size, state_size)
- Mamba2: conv = [x, B, C], temporal = (num_heads, head_dim)
- GDN (Gated Delta Net): conv = [Q, K, V] (dim(Q)==dim(K)),
temporal = (num_v_heads, v_dim, k_dim)
"""
import math
from dataclasses import dataclass
import torch
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
@dataclass(frozen=True)
class MambaConvSplitInfo:
"""Per-rank byte sizes of the conv sub-projections.
Used by both P and D sides for NIXL descriptor registration.
All fields are LOCAL to this engine's TP (already divided by TP size).
DS memory layout within one page (contiguous):
Mamba1: |---- x ----| (single sub-projection, no decomposition)
Mamba2: |-- x --|- B -|- C -| (B == C)
GDN: |- Q -|- K -|-- V --| (dim(Q)==dim(K), V may differ)
"""
conv_rows: int # conv_kernel - 1 (typically 3)
# Per-rank column counts per sub-projection:
# 1 entry for Mamba1, 3 for Mamba2/GDN.
local_proj_dims: tuple[int, ...]
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes)
@property
def local_conv_dim(self) -> int:
"""Total conv columns per rank."""
return sum(self.local_proj_dims)
@property
def proj_bytes(self) -> tuple[int, ...]:
"""Byte sizes of the sub-projections for one rank."""
row_bytes = self.conv_rows * self.conv_dtype_size
return tuple(d * row_bytes for d in self.local_proj_dims)
@property
def local_conv_offsets(self) -> list[tuple[int, int]]:
"""(byte_offset, byte_size) of each sub-projection within this
engine's page.
Used by both P and D for local descriptor registration.
"""
offsets: list[tuple[int, int]] = []
offset = 0
for size in self.proj_bytes:
offsets.append((offset, size))
offset += size
return offsets
def remote_conv_offsets(
self, local_rank_offset: int, tp_ratio: int
) -> list[tuple[int, int]]:
"""(byte_offset, byte_size) of this D rank's sub-projection slices
within one P page.
Used by D side only, during remote descriptor registration.
Args:
local_rank_offset: which slice this D rank reads.
tp_ratio: signed TP ratio.
>= 1: D_TP >= P_TP — P page is larger, D reads its slice.
< 0: P_TP > D_TP — P pages are smaller, D reads entire
P page. Local dims are scaled down by |tp_ratio|
to get P-sized offsets.
"""
offsets: list[tuple[int, int]] = []
if tp_ratio >= 1:
remote_base = 0
for size in self.proj_bytes:
offsets.append((remote_base + local_rank_offset * size, size))
remote_base += size * tp_ratio
else:
# NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages
# are smaller than D's. Local dims are D-sized, but we need
# P-sized offsets. Scale down by |tp_ratio|.
abs_ratio = -tp_ratio
remote_base = 0
for size in self.proj_bytes:
remote_size = size // abs_ratio
offsets.append((remote_base, remote_size))
remote_base += remote_size
return offsets
def derive_mamba_conv_split(
mamba_spec: MambaSpec,
local_tp: int,
) -> MambaConvSplitInfo:
"""Derive per-rank sub-projection byte sizes from a MambaSpec.
Called once at init on both P and D. Decomposes the conv dimension
into its sub-projection parts based on the model type.
Args:
mamba_spec: MambaSpec whose shapes are:
shapes[0] = conv state: (conv_dim_local, conv_rows) in DS layout.
shapes[1] = temporal state (model-specific shape).
local_tp: this engine's tensor-parallel size.
Returns:
MambaConvSplitInfo with per-rank sub-projection dims, conv_rows,
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
"""
_supported = (
MambaAttentionBackendEnum.MAMBA1,
MambaAttentionBackendEnum.MAMBA2,
MambaAttentionBackendEnum.GDN_ATTN,
)
if mamba_spec.mamba_type not in _supported:
raise NotImplementedError(
f"Conv transfer only supports Mamba1, Mamba2 and GDN models, "
f"got mamba_type={mamba_spec.mamba_type!r}."
)
conv_shape = mamba_spec.shapes[0]
assert len(conv_shape) == 2, f"Expected 2D conv state shape, got {conv_shape}"
# NOTE (ZhanqiuHu): 3-read requires DS layout, which is already asserted
# in nixl worker __init__. Use it directly instead of heuristic detection.
assert is_conv_state_dim_first(), "3-read requires DS conv state layout"
local_conv_dim = conv_shape[0] # DS: (conv_dim_local, conv_rows)
conv_rows = conv_shape[1]
conv_dtype_size = torch.tensor(
[],
dtype=mamba_spec.dtypes[0], # type: ignore[misc]
).element_size()
ssm_dtype_size = torch.tensor(
[],
dtype=mamba_spec.dtypes[1], # type: ignore[misc]
).element_size()
conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size
ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size
local_proj_dims: tuple[int, ...]
if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA1:
# Mamba1 conv state holds only x (no B/C), so it's a single
# contiguous TP shard with no sub-projection decomposition.
temporal_shape = mamba_spec.shapes[1]
assert temporal_shape[0] == local_conv_dim, (
f"Mamba1 temporal state dim ({temporal_shape[0]}) doesn't match "
f"conv dim ({local_conv_dim}); both should be "
f"intermediate_size/TP."
)
local_proj_dims = (local_conv_dim,)
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2:
# NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored
# in MambaSpec, so we reconstruct it from the SSM temporal state shape:
# shapes[1] = (local_num_heads, head_dim), already divided by TP.
head_dim = mamba_spec.shapes[1][1]
local_num_heads = mamba_spec.shapes[1][0]
intermediate_size = local_num_heads * local_tp * head_dim
# NOTE (ZhanqiuHu): global conv dim = intermediate_size + 2 * groups_ss,
# where groups_ss is the B (= C) dimension. B and C are always the same
# size, so we recover groups_ss from the remainder after subtracting x.
remainder = local_conv_dim * local_tp - intermediate_size
assert remainder > 0 and remainder % 2 == 0, (
f"Conv dim ({local_conv_dim}*tp={local_tp}) doesn't decompose "
f"into intermediate_size={intermediate_size} + 2*groups_ss. "
f"remainder={remainder}"
)
groups_ss = remainder // 2
# Divide by TP to get per-rank column counts.
x_local = intermediate_size // local_tp
b_local = groups_ss // local_tp
local_proj_dims = (x_local, b_local, b_local)
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.GDN_ATTN:
# GDN: conv = [Q, K, V] where dim(Q) == dim(K) == key_dim.
# conv_dim = key_dim*2 + value_dim (all global, divided by TP).
# Temporal state shape is (num_v_heads/TP, head_v_dim, head_k_dim).
temporal_shape = mamba_spec.shapes[1]
num_v_heads_local = temporal_shape[0]
head_v_dim = temporal_shape[1]
value_dim_local = num_v_heads_local * head_v_dim
remainder = local_conv_dim - value_dim_local
assert remainder > 0 and remainder % 2 == 0, (
f"GDN conv dim ({local_conv_dim}) doesn't decompose into "
f"2*key_dim_local + value_dim_local={value_dim_local}. "
f"remainder={remainder}"
)
key_dim_local = remainder // 2
local_proj_dims = (key_dim_local, key_dim_local, value_dim_local)
else:
raise NotImplementedError(
f"Conv split not supported for mamba_type={mamba_spec.mamba_type!r}"
)
return MambaConvSplitInfo(
conv_rows=conv_rows,
local_proj_dims=local_proj_dims,
conv_dtype_size=conv_dtype_size,
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
)
def compute_physical_blocks_per_logical(
ssm_sizes: tuple[int, ...], block_len: int
) -> int:
"""Derive _physical_blocks_per_logical_kv_block from remote metadata.
The remote engine's ratio is not sent directly in the handshake, so we
reconstruct it: total mamba state per logical block / block_len.
Args:
ssm_sizes: (conv_state_bytes, ssm_state_bytes) from NixlAgentMetadata.
block_len: the engine's block_len in bytes (from block_lens[0]).
"""
return math.ceil((ssm_sizes[0] + ssm_sizes[1]) / block_len)
@@ -1,459 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""IPC-based weight transfer engine using CUDA IPC for communication."""
import pickle
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, Any, ClassVar, Self
import pybase64 as base64
import torch
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
from vllm import envs
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
PackedBufferImporter,
packed_ipc_consumer,
packed_ipc_producer,
)
@dataclass
class IPCWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side init info for IPC weight transfer. No rendezvous needed.
`packed` is a must-agree wire param: the trainer ships it here at the init
handshake so the worker decodes with the same setting the trainer encoded
with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does
not need the buffer size (producer-only)."""
packed: bool = False
@dataclass
class IPCTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for IPC weight transfer. No rendezvous needed;
`rank` (from `TrainerInitInfo`) identifies this trainer process — rank 0
ships the merged IPC handles. All ranks still join the handle all-gather.
`packed` / `packed_buffer_size_bytes` are the transfer's wire params. The
trainer propagates `packed` to the worker at `trainer_init` so the two sides
cannot disagree. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "ipc"
packed: bool = False
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
@dataclass
class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Per-round update info for the IPC weight transfer backend."""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
ipc_handles: list[dict[str, tuple]] | dict[str, tuple] | None = None
"""IPC handles mapping physical GPU UUID to rebuild_cuda_tensor args.
For non-packed mode: list of per-parameter handle dicts.
For packed mode: single handle dict for the packed buffer."""
ipc_handles_pickled: str | None = None
"""Base64-encoded pickled IPC handles, used for HTTP transport."""
tensor_sizes: list[int] | None = None
"""Per-parameter sizes in bytes within the packed buffer.
Required when packed=True, unused otherwise."""
def __post_init__(self):
if self.ipc_handles_pickled is not None:
if self.ipc_handles is not None:
raise ValueError(
"Cannot specify both `ipc_handles` and `ipc_handles_pickled`"
)
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
raise ValueError(
"Refusing to deserialize `ipc_handles_pickled` without "
"VLLM_ALLOW_INSECURE_SERIALIZATION=1"
)
self.ipc_handles = pickle.loads(base64.b64decode(self.ipc_handles_pickled))
self.ipc_handles_pickled = None
if self.ipc_handles is None:
raise ValueError(
"Either `ipc_handles` or `ipc_handles_pickled` must be provided"
)
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
f"`dtype_names` should be of the same size as `names`: "
f"got {len(self.dtype_names)} and {len(self.names)}"
)
if len(self.shapes) != num_params:
raise ValueError(
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
# Unpacked transfers carry a per-parameter handle dict (a list); packed
# transfers carry a single dict for the whole buffer, so the list check
# only applies to the list case.
if isinstance(self.ipc_handles, list) and len(self.ipc_handles) != num_params:
raise ValueError(
f"`ipc_handles` should be of the same size as `names`: "
f"got {len(self.ipc_handles)} and {len(self.names)}"
)
class IPCWeightTransferEngine(
WeightTransferEngine[IPCWeightTransferInitInfo, IPCWeightTransferUpdateInfo]
):
"""Weight transfer engine using CUDA IPC between trainer and workers.
This implementation uses CUDA IPC to transfer weights from the trainer (rank 0)
to all inference workers in a process group. IPC handles are used to share
memory between processes on the same node.
"""
# Define backend-specific dataclass types
init_info_cls = IPCWeightTransferInitInfo
update_info_cls = IPCWeightTransferUpdateInfo
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
# Set from the trainer-supplied init info at the handshake; defaults are
# only for the (unreachable) receive-before-init case.
self.packed = False
# Shared across all chunks of every packed transfer this engine
# receives; see PackedBufferImporter for the refcount contract.
self._packed_importer = PackedBufferImporter()
def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None:
"""Initialize the weight transfer mechanism. No data-plane rendezvous is
needed for IPC; this just records the trainer-supplied wire params so the
worker decodes exactly as the trainer encoded.
Args:
init_info: IPC initialization info (carries `packed`).
"""
self.packed = init_info.packed
def start_weight_update(self) -> None:
"""Initialize layerwise reloading for the incoming checkpoint weights."""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading after all weights have been received."""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
finalize_layerwise_reload(self.model, self.model_config)
# Every reduce_tensor call is a fresh export with its own refcount
# slot, so releasing once per update always balances this update's
# export and lets the trainer reclaim its staging buffer. Callers
# that skip finish are still covered by the replace-on-next-export
# path inside the importer.
self._packed_importer.close()
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
"""Receive weights from the trainer via CUDA IPC handles and load them.
Whether the transfer is packed is read from `self.packed`, set at the
init handshake from the trainer's init info, so it is guaranteed to
match how the trainer encoded.
Args:
update_info: IPC update info containing parameter names, dtypes, shapes,
and IPC handles. Each IPC handle is a mapping between physical
GPU UUID and the rebuild_cuda_tensor args tuple.
"""
# Use the worker's assigned device rather than the ambient current
# device: the receive path is no longer wrapped in
# `with torch.device(self.device)` by the caller, so the current device
# is not guaranteed to match self.device. The IPC tensors must be
# rebuilt on the device the model lives on.
device_index = self.device.index
if self.packed:
if update_info.tensor_sizes is None:
raise ValueError("`tensor_sizes` is required when packed=True")
assert isinstance(update_info.ipc_handles, dict)
weights = packed_ipc_consumer(
ipc_handle=update_info.ipc_handles,
names=update_info.names,
shapes=update_info.shapes,
dtype_names=update_info.dtype_names,
tensor_sizes=update_info.tensor_sizes,
device_index=device_index,
importer=self._packed_importer,
)
else:
assert isinstance(update_info.ipc_handles, list)
weights = []
for name, ipc_handle in zip(
update_info.names,
update_info.ipc_handles,
):
props = torch.cuda.get_device_properties(device_index)
physical_gpu_id = str(props.uuid)
if physical_gpu_id not in ipc_handle:
raise ValueError(
f"IPC handle not found for GPU UUID "
f"{physical_gpu_id}. "
f"Available UUIDs: {list(ipc_handle.keys())}"
)
args = ipc_handle[physical_gpu_id]
list_args = list(args)
# Index 6 of the args from reduce_tensor is the device_index.
# We need to overwrite it with the receiver's device index.
list_args[6] = device_index
weight = rebuild_cuda_tensor(*list_args)
weights.append((name, weight))
from vllm.model_executor.model_loader.mtp_validation import (
disable_mtp_completeness_check,
)
with disable_mtp_completeness_check():
self.model.load_weights(weights)
def shutdown(self) -> None:
self._packed_importer.close()
class IPCTrainerWeightTransferEngine(TrainerWeightTransferEngine[IPCTrainerInitInfo]):
"""Trainer-side CUDA IPC weight transfer engine.
Called on every trainer rank. For multi-rank (e.g. FSDP) trainers all ranks
iterate the source (materializing each tensor) and contribute to the
IPC-handle all-gather; only the sender (rank 0) ships the merged handles to
the inference side. IPC transfer is straight-line (no concurrent broadcast
like NCCL): `update_weights` *is* the transfer, and it rides the client, so
it no-ops on non-senders.
`packed` / `packed_buffer_size_bytes` come from `IPCTrainerInitInfo`; the
sender propagates `packed` to the worker at `trainer_init`.
"""
init_info_cls = IPCTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: WeightSource,
is_sender: bool = True,
packed: bool = False,
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
) -> None:
super().__init__(client=client, source=source, is_sender=is_sender)
self.packed = packed
self.packed_buffer_size_bytes = packed_buffer_size_bytes
self.device_index = torch.accelerator.current_device_index()
self.gpu_uuid = str(torch.cuda.get_device_properties(self.device_index).uuid)
@classmethod
def trainer_init(
cls,
init_info: IPCTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
if source is None:
raise ValueError("IPC trainer weight transfer requires a WeightSource.")
engine = cls(
client=client,
source=source,
is_sender=init_info.is_sender,
packed=init_info.packed,
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
)
# IPC needs no data-plane rendezvous. The sender ships the must-agree
# `packed` flag so the worker decodes exactly as this trainer encodes.
if engine.is_sender:
engine.client.init_weight_transfer_engine({"packed": init_info.packed})
return engine
def send_weights(self) -> None:
assert self.source is not None
source = self.source
if self.is_sender:
self.client.start_weight_update()
# Unpacked returns strong refs to its IPC-shared copies; they must stay
# alive across `finish_weight_update` too.
weight_refs = self._send(source)
if self.is_sender:
self.client.finish_weight_update()
self._post_send_sync()
del weight_refs
# ---- data plane (runs on all ranks; only the sender ships) ----
def _send(self, source: WeightSource) -> list[torch.Tensor] | None:
if self.packed:
self._send_packed(source)
return None
return self._send_unpacked(source)
def _all_gather_and_merge_handles(
self,
handles: list[dict[str, tuple]],
) -> list[dict[str, tuple]]:
"""All-gather and merge IPC handle dicts across ranks in one call.
Each rank contributes a list of {gpu_uuid: ipc_args} dicts (one
per parameter or one per chunk). A single all_gather_object
collects every rank's full list, then the sender merges per-index so
each dict maps every GPU UUID to its args.
The all-gather runs over the *default* process group; this assumes the
default group is exactly the set of colocated trainer ranks and that the
sender is a member. No-op (returns handles unchanged) when no distributed
group exists.
"""
if (
not torch.distributed.is_initialized()
or torch.distributed.get_world_size() == 1
):
return handles
world_size = torch.distributed.get_world_size()
gathered: list[list[dict[str, tuple]] | None] = [None] * world_size
torch.distributed.all_gather_object(gathered, handles)
torch.distributed.barrier()
torch.cuda.synchronize()
if self.is_sender:
merged: list[dict[str, tuple]] = []
for param_idx in range(len(handles)):
m: dict[str, tuple] = {}
for rank_handles in gathered:
if rank_handles is not None:
m.update(rank_handles[param_idx])
merged.append(m)
return merged
return [{} for _ in handles]
@staticmethod
def _post_send_sync() -> None:
"""Barrier + ipc_collect after a send; no-op if single-GPU."""
if (
torch.distributed.is_initialized()
and torch.distributed.get_world_size() > 1
):
torch.distributed.barrier()
torch.cuda.ipc_collect()
def _send_unpacked(self, source: WeightSource) -> list[torch.Tensor]:
"""Iterate the source, build one IPC handle per param, all-gather the
handles across ranks, and (sender) ship them in one update call.
Returns the strong refs to every contiguous copy. reduce_tensor's args
do NOT keep storage alive, and non-contiguous inputs allocate fresh
storage in .contiguous(); the caller must keep these alive until the
post-send barrier (past `finish`) so the consumer's IPC views stay valid.
"""
names: list[str] = []
dtype_names: list[str] = []
shapes: list[list[int]] = []
ipc_handles: list[dict[str, tuple]] = []
weight_refs: list[torch.Tensor] = []
for name, tensor in source:
# FSDP shards were gathered by the source; ensure contiguity here.
names.append(name)
dtype_names.append(str(tensor.dtype).split(".")[-1])
shapes.append(list(tensor.shape))
weight = tensor.detach().contiguous()
weight_refs.append(weight)
_, ipc_args = reduce_tensor(weight)
ipc_handles.append({self.gpu_uuid: ipc_args})
ipc_handles = self._all_gather_and_merge_handles(ipc_handles)
self._do_send(
names=names,
dtype_names=dtype_names,
shapes=shapes,
ipc_handles=ipc_handles,
)
return weight_refs
def _send_packed(self, source: WeightSource) -> None:
"""Send weights in bounded-memory chunks (packed mode)."""
for chunk in packed_ipc_producer(
iterator=iter(source),
gpu_uuid=self.gpu_uuid,
post_iter_func=lambda item: item[1],
buffer_size_bytes=self.packed_buffer_size_bytes,
):
ipc_handle = self._all_gather_and_merge_handles([chunk.ipc_handle])[0]
self._do_send(
names=chunk.names,
dtype_names=chunk.dtype_names,
shapes=chunk.shapes,
ipc_handles=ipc_handle,
tensor_sizes=chunk.tensor_sizes,
)
# Per-chunk barrier: the producer reuses a single IPC buffer across
# chunks, but only the sender waits for the consumers (via _do_send).
# Without syncing every rank here, non-sender ranks race ahead and
# overwrite their buffer while their colocated worker is still
# reading the current chunk, silently corrupting the transfer.
self._post_send_sync()
def _do_send(
self,
names: list[str],
dtype_names: list[str],
shapes: list[list[int]],
ipc_handles: list[dict[str, tuple]] | dict[str, tuple],
tensor_sizes: list[int] | None = None,
) -> None:
"""Build one update payload and ship it via the client. Only the sender
ships (non-sender ranks already contributed to the handle all-gather).
Emits raw `ipc_handles`; transports that cannot carry them natively
(HTTP/JSON) pickle them in their client (see `HTTPVLLMWeightSyncClient`).
"""
if not self.is_sender:
return
update_fields: dict[str, Any] = {
"names": names,
"dtype_names": dtype_names,
"shapes": shapes,
"ipc_handles": ipc_handles,
}
if tensor_sizes is not None:
update_fields["tensor_sizes"] = tensor_sizes
update_info = IPCWeightTransferUpdateInfo(**update_fields)
self.client.update_weights(asdict(update_info))
@@ -1,864 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared helpers for the NCCL M2N (`nccl_m2n`) weight transfer backend.
M2N reshards a tensor between two disjoint meshes of ranks that live in one
communicator: the trainer occupies ranks `[0, T)` and the inference workers
`[T, T + N)`. This module holds everything both sides need — the optional
runtime import, layout descriptors, and the conversion to `nccl.m2n` types —
so the engine module stays about the transfer itself.
"""
import hashlib
import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
import torch
from vllm.distributed.weight_transfer.base import ParamMeta
from vllm.utils.nccl import unpinned_nccl_env
if TYPE_CHECKING:
from vllm.distributed.utils import StatelessProcessGroup
# Placement code for a mesh axis that replicates. Any other (non-negative) code
# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so
# layouts survive the JSON init handshake without a custom encoder.
REPLICATE = -1
# `NCCL_RESHARD_MAX_TENSOR_DIMS`; 4-D and higher are rejected by the library.
MAX_TENSOR_DIMS = 3
MESH_NDIMS = 2
# Destination mesh convention. Keeping these roles named avoids scattering
# positional axis assumptions through the worker-side planner.
DESTINATION_REPLICA_AXIS = 0
DESTINATION_SHARD_AXIS = 1
# Conservative capacities accepted by the supported nccl_m2n algorithms. They
# are not exposed by its Python API yet; remove these mirrors once nccl_m2n
# provides communication-free plan validation.
MAX_SOURCE_SHARDS = 16
MAX_DEST_SHARDS = 64
# Increment when the JSON init payload changes incompatibly.
M2N_WIRE_SCHEMA_VERSION = 2
SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset(
dtype
for dtype in (
getattr(torch, name, None)
for name in (
"int8",
"uint8",
"float8_e4m3fn",
"float8_e5m2",
"float16",
"bfloat16",
"int32",
"uint32",
"float32",
"int64",
"uint64",
"float64",
)
)
if dtype is not None
)
_IMPORT_HINT = (
"The nccl_m2n weight transfer backend requires the `nccl-extensions` "
"package (and its `nccl4py` dependency), which is not installed by vLLM. "
"See https://github.com/NVIDIA/nccl-extensions for build "
"and install instructions. It needs NCCL 2.30.5 or newer."
)
def prepare_m2n_local_runtime(m2n: Any, max_cta: int | None) -> tuple[Any, int, Any]:
"""Prepare rank-local resources before any rank enters NCCL init."""
import nccl.core as nccl
device = torch.accelerator.current_device_index()
if not isinstance(device, int) or isinstance(device, bool) or device < 0:
raise RuntimeError(f"invalid current accelerator device index: {device!r}")
handle = m2n.Handle.create(m2n.Config(max_cta=max_cta))
return nccl, device, handle
def create_m2n_communicator(
nccl: Any,
group: "StatelessProcessGroup",
unique_id_bytes: bytes | None,
) -> Any:
"""Create the M2N data-plane communicator through NCCL4Py."""
if unique_id_bytes is None:
envelope: dict[str, bytes | str | None] | None = None
if group.rank == 0:
try:
envelope = {
"unique_id": bytes(nccl.get_unique_id().as_bytes),
"error": None,
}
except Exception as exc:
envelope = {
"unique_id": None,
"error": f"{type(exc).__name__}: {exc}",
}
envelope = group.broadcast_obj(envelope, src=0)
if not isinstance(envelope, Mapping) or set(envelope) != {"unique_id", "error"}:
raise RuntimeError("invalid NCCL4Py unique-id envelope")
error = envelope["error"]
received_unique_id = envelope["unique_id"]
if error is not None:
if not isinstance(error, str) or not error:
raise RuntimeError("invalid NCCL4Py unique-id error")
raise RuntimeError(f"NCCL4Py unique-id generation failed: {error}")
if not isinstance(received_unique_id, bytes):
raise RuntimeError("NCCL4Py unique-id broadcast did not return bytes")
unique_id_bytes = received_unique_id
unique_id = nccl.UniqueId.from_bytes(unique_id_bytes)
with unpinned_nccl_env():
return nccl.Communicator.init(
nranks=group.world_size,
rank=group.rank,
unique_id=unique_id,
)
def import_m2n() -> Any:
"""Import `nccl.m2n` lazily, with an actionable error when it is missing.
Deferred so that importing vLLM — or any other weight transfer backend —
never requires the m2n runtime to be present.
"""
try:
import nccl.m2n as m2n
except ImportError as e:
raise ImportError(f"{_IMPORT_HINT} (import failed: {e})") from e
return m2n
@dataclass(frozen=True)
class M2NMesh:
"""One side's rank topology — pure topology, no tensor placement.
Mirrors `ncclMesh_t`: a 2-axis mesh owning the contiguous rank interval
`[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a
single-axis topology is spelled with a second axis of size 1.
A model may use a different factorization for each tensor. For example,
dense weights can use a DP x TP mesh while expert weights use EDP x EP.
"""
dims: tuple[int, int]
start_rank: int
def __post_init__(self) -> None:
object.__setattr__(self, "dims", tuple(self.dims))
if len(self.dims) != MESH_NDIMS or any(
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
for dim in self.dims
):
raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints")
if (
not isinstance(self.start_rank, int)
or isinstance(self.start_rank, bool)
or self.start_rank < 0
):
raise ValueError(
f"mesh start_rank must be non-negative, got {self.start_rank}"
)
@property
def size(self) -> int:
return self.dims[0] * self.dims[1]
# A tensor's placement over its side's mesh: one code per mesh axis
# (`REPLICATE`, or the tensor dim that axis shards), or `REPLICATED` for a
# tensor every rank holds in full.
Placements = tuple[int, int]
REPLICATED: Placements | None = None
@dataclass(frozen=True)
class M2NLayout:
"""A tensor's mesh and placement on one side of a transfer."""
mesh: M2NMesh
placements: Placements | None
def __post_init__(self) -> None:
if not isinstance(self.mesh, M2NMesh):
raise TypeError("M2N layout mesh must be an M2NMesh")
if self.placements is not None:
object.__setattr__(self, "placements", tuple(self.placements))
def check_placements(placements: Placements, context: str = "placements") -> None:
"""Reject placement pairs nccl_m2n cannot express."""
if len(placements) != MESH_NDIMS:
raise ValueError(f"{context} must have {MESH_NDIMS} entries")
if any(not isinstance(code, int) or isinstance(code, bool) for code in placements):
raise TypeError(f"{context} must contain integer placement codes")
invalid = [code for code in placements if code < REPLICATE]
if invalid:
raise ValueError(
f"{context} contains invalid placement code {invalid[0]}; valid "
f"codes are {REPLICATE} (Replicate) and non-negative tensor dimensions"
)
num_sharded = sum(code != REPLICATE for code in placements)
if num_sharded == 0:
raise ValueError(
f"{context}: a fully replicated tensor is carried as REPLICATED, "
"not as two REPLICATE axes"
)
if num_sharded == MESH_NDIMS:
raise ValueError(
f"{context}: nccl_m2n needs one REPLICATE mesh axis, but "
f"{placements} shards both"
)
def resolve_layout(
mesh: M2NMesh,
placements: Placements | None,
context: str = "placements",
) -> tuple[M2NMesh, Placements]:
"""Pair one tensor's placement with the mesh m2n should see for it.
A replicated tensor needs a size-1 mesh axis to carry a no-op shard, since
m2n has no `{REPLICATE, REPLICATE}`. It is therefore described over the
*same rank interval* re-factored as `(size, 1)`. That re-factoring is sound
precisely because replication is order-independent — every rank holds the
whole tensor, so it does not matter that `(a, b)` and `(size, 1)` walk the
interval in a different order. A sharded tensor keeps its side's own
factorization, where rank order decides who owns which shard.
"""
if placements is None:
return M2NMesh((mesh.size, 1), mesh.start_rank), (REPLICATE, 0)
check_placements(placements, context)
return mesh, placements
def validate_layout(
mesh: M2NMesh,
placements: Placements,
shape: Sequence[int],
context: str,
) -> tuple[int, ...]:
"""Validate one resolved layout and return its required local shape."""
local_shape = list(shape)
for axis, tensor_dim in enumerate(placements):
if tensor_dim == REPLICATE:
continue
if tensor_dim >= len(shape):
raise ValueError(
f"{context} shards tensor dim {tensor_dim}, but parameter has "
f"rank {len(shape)}"
)
factor = mesh.dims[axis]
if shape[tensor_dim] % factor:
raise ValueError(
f"{context} shards dim {tensor_dim} (size {shape[tensor_dim]}) "
f"over {factor} ranks, which does not divide evenly"
)
local_shape[tensor_dim] //= factor
return tuple(local_shape)
def expected_local_shape(meta: "M2NParamMeta") -> tuple[int, ...]:
"""Validate a parameter's source layout and return its local shape."""
mesh, placements = resolve_layout(
meta.source_layout.mesh,
meta.source_layout.placements,
f"parameter '{meta.name}' source placements",
)
return validate_layout(
mesh, placements, meta.shape, f"parameter '{meta.name}' source layout"
)
def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None:
"""Reject parameters nccl_m2n cannot transfer."""
if dtype not in SUPPORTED_DTYPES:
raise ValueError(
f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not "
f"support. Supported: {sorted(str(d) for d in SUPPORTED_DTYPES)}"
)
if not 1 <= len(shape) <= MAX_TENSOR_DIMS:
raise ValueError(
f"parameter '{name}' has rank {len(shape)}; nccl_m2n supports "
f"rank 1..{MAX_TENSOR_DIMS}"
)
def _shard_spec(mesh: M2NMesh, placements: Placements) -> tuple[int, int]:
return next(
(code, mesh.dims[axis])
for axis, code in enumerate(placements)
if code != REPLICATE
)
def _max_partition_overlaps(
extent: int,
partitions: int,
other_partitions: int,
) -> int:
"""Return the maximum other partitions intersected by any one partition."""
if extent == 0:
return 0
width = extent // partitions
other_width = extent // other_partitions
max_overlaps = 0
for index in range(partitions):
start = index * width
end = start + width
first_other = start // other_width
last_other = (end - 1) // other_width
max_overlaps = max(max_overlaps, last_other - first_other + 1)
return max_overlaps
def check_plan_limits(
src: tuple[M2NMesh, Placements],
dst: tuple[M2NMesh, Placements],
name: str,
shape: Sequence[int],
) -> None:
"""Reject plans whose per-shard fan-in or fan-out exceeds M2N capacity."""
src_dim, src_shards = _shard_spec(*src)
dst_dim, dst_shards = _shard_spec(*dst)
if any(extent == 0 for extent in shape):
max_sources = max_targets = 0
elif src_dim != dst_dim:
max_sources = src_shards
max_targets = dst_shards
else:
extent = shape[src_dim]
max_sources = _max_partition_overlaps(extent, dst_shards, src_shards)
max_targets = _max_partition_overlaps(extent, src_shards, dst_shards)
if max_sources > MAX_SOURCE_SHARDS:
raise ValueError(
f"parameter '{name}' would feed up to {max_sources} source shards "
f"into one destination shard, over nccl_m2n MAX_SOURCES="
f"{MAX_SOURCE_SHARDS}"
)
if max_targets > MAX_DEST_SHARDS:
raise ValueError(
f"parameter '{name}' would fan one source shard out to up to "
f"{max_targets} destination shards, over nccl_m2n "
f"MAX_TARGETS={MAX_DEST_SHARDS}"
)
def validate_transfer_plan(
metas: Sequence["M2NParamMeta"],
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
) -> None:
"""Validate the complete ordered source-to-destination transfer plan."""
if len(dst_placements) != len(metas):
raise ValueError(
f"destination plan has {len(dst_placements)} parameters, expected "
f"{len(metas)}"
)
for meta, placement in zip(metas, dst_placements):
check_transferable(meta.name, meta.dtype, meta.shape)
src = resolve_layout(
meta.source_layout.mesh,
meta.source_layout.placements,
f"parameter '{meta.name}' source placements",
)
validate_layout(*src, meta.shape, f"parameter '{meta.name}' source layout")
dst = resolve_layout(
dst_mesh,
placement,
f"parameter '{meta.name}' destination placements",
)
validate_layout(*dst, meta.shape, f"parameter '{meta.name}' destination layout")
check_plan_limits(src, dst, meta.name, meta.shape)
@dataclass(frozen=True)
class M2NParamMeta(ParamMeta):
"""`ParamMeta` extended with how the trainer places this tensor.
The base class carries only name / dtype / full shape, which is not enough
to plan a reshard. `source_layout` carries both the per-parameter mesh and
its placement on that mesh.
"""
source_layout: M2NLayout
def __post_init__(self) -> None:
if not isinstance(self.source_layout, M2NLayout):
raise TypeError("source_layout must be an M2NLayout")
@dataclass(frozen=True)
class M2NWireParam:
"""JSON-safe source metadata for one parameter in the init handshake."""
name: str
dtype_name: str
shape: tuple[int, ...]
src_mesh_dims: tuple[int, int]
src_placements: Placements | None
def __post_init__(self) -> None:
shape = tuple(self.shape)
dims = tuple(self.src_mesh_dims)
placements = None if self.src_placements is None else tuple(self.src_placements)
object.__setattr__(self, "shape", shape)
object.__setattr__(self, "src_mesh_dims", dims)
object.__setattr__(self, "src_placements", placements)
if not isinstance(self.name, str) or not self.name:
raise ValueError("M2N wire parameter name must not be empty")
if not isinstance(self.dtype_name, str) or not self.dtype_name:
raise ValueError(f"parameter '{self.name}' has an empty wire dtype name")
if not shape or any(
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
for dim in shape
):
raise ValueError(
f"parameter '{self.name}' shape must contain positive integers"
)
M2NMesh(cast(tuple[int, int], dims), 0)
if placements is not None:
check_placements(
cast(Placements, placements),
f"parameter '{self.name}' source placements",
)
@classmethod
def from_dict(cls, value: Mapping[str, Any]) -> "M2NWireParam":
"""Parse one strict wire record from its JSON-decoded mapping."""
fields = {
"name",
"dtype_name",
"shape",
"src_mesh_dims",
"src_placements",
}
missing = fields - value.keys()
extra = value.keys() - fields
if missing or extra:
details = []
if missing:
details.append(f"missing {sorted(missing)}")
if extra:
details.append(f"unexpected {sorted(extra)}")
raise ValueError(f"invalid M2N wire parameter: {', '.join(details)}")
return cls(**value)
def to_dict(self) -> dict[str, Any]:
"""Serialize this record using only JSON-compatible values."""
return {
"name": self.name,
"dtype_name": self.dtype_name,
"shape": list(self.shape),
"src_mesh_dims": list(self.src_mesh_dims),
"src_placements": (
None if self.src_placements is None else list(self.src_placements)
),
}
def source_plan_digest(params: Sequence[M2NWireParam]) -> str:
"""Return a rank-independent digest of the ordered source wire plan."""
payload = json.dumps(
{
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"params": [param.to_dict() for param in params],
},
sort_keys=True,
separators=(",", ":"),
).encode()
return hashlib.sha256(payload).hexdigest()
def complete_plan_digest(
source_digest: str,
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
) -> str:
"""Return a digest of the agreed source and destination plan."""
if not isinstance(source_digest, str) or not source_digest:
raise ValueError("source digest must be a non-empty string")
payload = json.dumps(
{
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"source_digest": source_digest,
"destination_mesh": {
"dims": list(dst_mesh.dims),
"start_rank": dst_mesh.start_rank,
},
"destination_placements": [
None if placement is None else list(placement)
for placement in dst_placements
],
},
sort_keys=True,
separators=(",", ":"),
).encode()
return hashlib.sha256(payload).hexdigest()
def check_source_plan_agreement(
group: "StatelessProcessGroup",
params: Sequence[M2NWireParam],
expected_digest: str | None = None,
local_error: str | None = None,
) -> str:
"""Verify every participant entered with the same source plan."""
prepared_error = local_error
digest: str | None = None
declared: str | None = expected_digest
if prepared_error is None:
try:
digest = source_plan_digest(params)
declared = digest if expected_digest is None else expected_digest
if not isinstance(declared, str) or not declared:
raise ValueError("declared source digest must be a non-empty string")
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local source error must be a non-empty string"
gathered = group.all_gather_obj(
{
"phase": "source",
"digest": digest,
"declared_digest": declared,
"error": prepared_error,
}
)
errors: list[str] = []
expected_fields = {"phase", "digest", "declared_digest", "error"}
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
agreed: list[tuple[int, str, str]] = []
for rank, item in enumerate(gathered):
if not isinstance(item, Mapping) or set(item) != expected_fields:
errors.append(f"rank {rank}: invalid source preflight envelope")
continue
if item["phase"] != "source":
errors.append(f"rank {rank}: invalid source preflight phase")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid source preflight error")
else:
errors.append(f"rank {rank}: {error}")
continue
computed = item["digest"]
claimed = item["declared_digest"]
if (
not isinstance(computed, str)
or not computed
or not isinstance(claimed, str)
or not claimed
):
errors.append(f"rank {rank}: invalid source digest")
continue
agreed.append((rank, computed, claimed))
if errors:
raise RuntimeError(
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
)
canonical = agreed[0][1]
for rank, computed, claimed in agreed:
if computed != claimed:
errors.append(
f"rank {rank}: computed digest {computed} does not match "
f"declared digest {claimed}"
)
if computed != canonical:
errors.append(f"rank {rank}: source digest disagrees with rank 0")
if errors:
raise RuntimeError(
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
)
return canonical
def check_complete_plan_agreement(
group: "StatelessProcessGroup",
source_digest: str,
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
local_error: str | None = None,
) -> str:
"""Require every rank to accept the same complete transfer plan."""
prepared_error = local_error
digest: str | None = None
if prepared_error is None:
try:
digest = complete_plan_digest(source_digest, dst_mesh, dst_placements)
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local plan error must be a non-empty string"
gathered = group.all_gather_obj(
{"phase": "complete_plan", "digest": digest, "error": prepared_error}
)
errors: list[str] = []
valid: list[tuple[int, str]] = []
expected_fields = {"phase", "digest", "error"}
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != expected_fields
or item.get("phase") != "complete_plan"
):
errors.append(f"rank {rank}: invalid complete-plan envelope")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid complete-plan error")
else:
errors.append(f"rank {rank}: {error}")
continue
rank_digest = item["digest"]
if not isinstance(rank_digest, str) or not rank_digest:
errors.append(f"rank {rank}: invalid complete-plan digest")
continue
valid.append((rank, rank_digest))
if not errors:
canonical = valid[0][1]
for rank, rank_digest in valid[1:]:
if rank_digest != canonical:
errors.append(
f"rank {rank}: complete-plan digest disagrees with rank 0"
)
if errors:
raise RuntimeError(
"nccl_m2n complete-plan preflight rejected before first reshard: "
+ "; ".join(errors)
)
return valid[0][1]
def check_runtime_ready_agreement(
group: "StatelessProcessGroup", local_error: str | None
) -> None:
"""Require every rank to prepare local resources before NCCL init."""
prepared_error = local_error
if prepared_error is not None and (
not isinstance(prepared_error, str) or not prepared_error
):
prepared_error = "TypeError: local runtime error must be a non-empty string"
gathered = group.all_gather_obj({"phase": "runtime_ready", "error": prepared_error})
errors: list[str] = []
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != {"phase", "error"}
or item.get("phase") != "runtime_ready"
):
errors.append(f"rank {rank}: invalid runtime readiness envelope")
continue
error = item.get("error")
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid runtime readiness error")
else:
errors.append(f"rank {rank}: {error}")
if errors:
raise RuntimeError(
"nccl_m2n local runtime preflight failed before NCCL init: "
+ "; ".join(errors)
)
def check_data_plane_agreement(
group: "StatelessProcessGroup",
unique_id_bytes: bytes | None,
local_error: str | None = None,
) -> None:
"""Agree on the NCCL rendezvous identity before NCCL init."""
prepared_error = local_error
uid_digest: str | None = None
if prepared_error is None:
try:
if unique_id_bytes is not None:
if not isinstance(unique_id_bytes, bytes):
raise TypeError("NCCL unique id must decode to bytes")
uid_digest = hashlib.sha256(unique_id_bytes).hexdigest()
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local data-plane error must be non-empty"
gathered = group.all_gather_obj(
{
"phase": "data_plane",
"mode": "uid" if unique_id_bytes is not None else "tcp",
"uid_digest": uid_digest,
"error": prepared_error,
}
)
expected_fields = {
"phase",
"mode",
"uid_digest",
"error",
}
errors: list[str] = []
valid: list[tuple[int, str, str | None]] = []
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != expected_fields
or item.get("phase") != "data_plane"
):
errors.append(f"rank {rank}: invalid data-plane preflight envelope")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid data-plane preflight error")
else:
errors.append(f"rank {rank}: {error}")
continue
mode = item["mode"]
uid = item["uid_digest"]
if mode not in {"tcp", "uid"} or (mode == "uid") != isinstance(uid, str):
errors.append(f"rank {rank}: invalid data-plane identity")
continue
valid.append((rank, mode, uid))
if not errors:
_, mode, uid = valid[0]
for rank, other_mode, other_uid in valid[1:]:
if (other_mode, other_uid) != (mode, uid):
errors.append(f"rank {rank}: NCCL data-plane identity disagrees")
if errors:
raise RuntimeError(
"nccl_m2n data-plane preflight rejected before NCCL init: "
+ "; ".join(errors)
)
def validate_local_buffer(m2n: Any, meta: M2NParamMeta, buffer: Any) -> None:
"""Check source dtype/shape while delegating buffer validity to M2N."""
# A raw pointer carries no inspectable metadata. The source owns its
# correctness; _send supplies the declared dtype and local shape to M2N.
if isinstance(buffer, int):
return
mesh, placements = resolve_layout(
meta.source_layout.mesh, meta.source_layout.placements
)
actual = m2n.DistTensor(
buffer,
mesh=to_mesh(m2n, mesh),
placements=to_placements(m2n, placements),
)
declared = m2n.DistTensor(
None,
local_shape=actual.local_shape,
dtype=meta.dtype,
mesh=to_mesh(m2n, mesh),
placements=to_placements(m2n, placements),
)
if actual.dtype != declared.dtype:
raise ValueError(
f"parameter '{meta.name}' source returned dtype {actual.dtype}, "
f"but its metadata declared {declared.dtype}"
)
expected = expected_local_shape(meta)
if actual.local_shape != expected:
raise ValueError(
f"parameter '{meta.name}' source returned local shape "
f"{actual.local_shape}, but layout {meta.source_layout} implies "
f"{expected} from global shape {meta.shape}"
)
def to_mesh(m2n: Any, mesh: M2NMesh) -> Any:
return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank)
def to_placements(m2n: Any, placements: Placements) -> list[Any]:
return [
m2n.Replicate() if code == REPLICATE else m2n.Shard(code) for code in placements
]
def publish_destination_placements(
comm: Any,
device: torch.device,
stream: torch.cuda.Stream,
first_worker_rank: int,
placements: "Sequence[Placements | None] | None",
num_parameters: int,
) -> list[Placements | None]:
"""Share the worker-side destination plan with every rank in the group.
The trainer must issue each reshard with the same destination the workers
use, but once destinations are per-parameter they depend on the inference
model, which only the workers can see. The first worker publishes them here,
over the shared NCCL communicator; trainer ranks pass `None` and receive
them, and the other workers pass their own so a disagreement is caught
rather than deadlocking later.
Only placements travel: the destination *mesh* is still derived from the
rank split, identically on both sides. The plan is published on the data
plane before any M2N reshard collective.
"""
replicated_sentinel = REPLICATE - 1
encoded = torch.full(
(num_parameters, MESH_NDIMS),
replicated_sentinel,
dtype=torch.int8,
device=device,
)
if comm.rank == first_worker_rank:
if placements is None or len(placements) != num_parameters:
raise ValueError(
f"publishing rank needs {num_parameters} destination placements"
)
rows = [
[replicated_sentinel] * MESH_NDIMS if placement is REPLICATED else placement
for placement in placements
]
if rows:
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=device))
comm.broadcast(encoded, encoded, root=first_worker_rank, stream=stream)
return [
REPLICATED
if all(code == replicated_sentinel for code in row)
else cast(Placements, tuple(row))
for row in encoded.tolist()
]
@@ -1,599 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-side weight transfer engine built on NCCL M2N (`nccl_m2n`).
The trainer and the inference workers share one NCCL communicator: trainer ranks
occupy `[0, T)`, workers `[T, T + N)`. Each parameter is moved with a single
`nccl.m2n.reshard`, which redistributes it from the trainer's layout (FSDP / EP
/ arbitrary DTensor sharding) to the inference layout — so the trainer sends its
local shards and never all-gathers a full tensor, which is what the broadcast
NCCL backend forces it to do.
When an incoming checkpoint parameter maps directly to a live vLLM parameter,
M2N reshards into that worker's local model storage. Parameters whose names or
layouts cannot be resolved receive a full tensor and fall back to
`load_weights`, preserving the existing backend's loading behavior.
Both meshes participate in every reshard, in the same order, so this backend has
the same concurrency shape as the broadcast NCCL backend: the worker must be
inside `receive_weights` while the trainer is sending. Driving that from the
trainer is the trainer engine's job.
"""
from collections.abc import Iterator, Mapping
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, cast
import torch
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.m2n_common import (
DESTINATION_SHARD_AXIS,
M2N_WIRE_SCHEMA_VERSION,
MESH_NDIMS,
M2NLayout,
M2NMesh,
M2NParamMeta,
M2NWireParam,
Placements,
check_complete_plan_agreement,
check_data_plane_agreement,
check_runtime_ready_agreement,
check_source_plan_agreement,
check_transferable,
create_m2n_communicator,
expected_local_shape,
import_m2n,
prepare_m2n_local_runtime,
publish_destination_placements,
resolve_layout,
source_plan_digest,
to_mesh,
to_placements,
validate_transfer_plan,
)
from vllm.distributed.weight_transfer.m2n_layout import (
M2NDestination,
resolve_parameter_destinations,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
decode_nccl_unique_id,
worker_init_metadata_group,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
__all__ = [
"M2NWeightTransferInitInfo",
"M2NWeightTransferUpdateInfo",
"M2NWeightTransferEngine",
]
# ---------------------------------------------------------------------------
# Wire types
# ---------------------------------------------------------------------------
@dataclass(kw_only=True)
class M2NWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side init info: the rendezvous plus the full transfer plan.
Layouts are static for the whole run, so they ride the one-time init
handshake and the per-round update info stays a list of names. Everything is
plain JSON so the HTTP control plane carries it unchanged.
"""
schema_version: int
master_address: str
master_port: int
rank_offset: int
"""First worker rank, i.e. the number of trainer ranks."""
world_size: int
"""Trainer ranks + all inference workers."""
dst_mesh_dims: list[int]
"""The inference mesh, starting at `rank_offset`. Declared rather than
derived so both sides describe the destination identically."""
source_digest: str
"""SHA-256 of the ordered, versioned source parameter plan."""
params: list[dict[str, Any]]
"""Ordered per-parameter source metadata and layouts."""
nccl_unique_id_b64: str | None = field(default=None, repr=False)
max_cta: int | None = None
def __post_init__(self) -> None:
if not isinstance(self.master_address, str) or not self.master_address:
raise ValueError("`master_address` must name the metadata rendezvous")
if (
not isinstance(self.master_port, int)
or isinstance(self.master_port, bool)
or not 0 < self.master_port < 65536
):
raise ValueError("`master_port` must be a valid metadata port")
if (
not isinstance(self.rank_offset, int)
or isinstance(self.rank_offset, bool)
or not isinstance(self.world_size, int)
or isinstance(self.world_size, bool)
or self.rank_offset < 1
or self.rank_offset >= self.world_size
):
raise ValueError(
f"`rank_offset` ({self.rank_offset}) must leave at least one "
f"trainer rank and one worker in world_size {self.world_size}"
)
num_workers = self.world_size - self.rank_offset
dst = tuple(self.dst_mesh_dims)
try:
dst_mesh = M2NMesh(cast(tuple[int, int], dst), self.rank_offset)
except (TypeError, ValueError) as exc:
raise ValueError(f"invalid `dst_mesh_dims` {dst}: {exc}") from exc
if dst_mesh.size != num_workers:
raise ValueError(
f"`dst_mesh_dims` {dst} must be {MESH_NDIMS} dims covering the "
f"{num_workers} inference workers"
)
def parse_wire_params(self) -> tuple[M2NWireParam, ...]:
"""Parse and authenticate the ordered source plan."""
if self.schema_version != M2N_WIRE_SCHEMA_VERSION:
raise ValueError(
"unsupported nccl_m2n init schema version "
f"{self.schema_version}; expected {M2N_WIRE_SCHEMA_VERSION}"
)
if not isinstance(self.params, list):
raise TypeError("`params` must be a list of wire parameter mappings")
if not isinstance(self.source_digest, str) or not self.source_digest:
raise ValueError("`source_digest` must be a non-empty string")
wire_params: list[M2NWireParam] = []
names: set[str] = set()
for index, value in enumerate(self.params):
if not isinstance(value, Mapping):
raise TypeError(
f"`params[{index}]` must be a mapping, got {type(value).__name__}"
)
param = M2NWireParam.from_dict(value)
if param.name in names:
raise ValueError(
f"nccl_m2n init params contain duplicate parameter '{param.name}'"
)
names.add(param.name)
wire_params.append(param)
parsed = tuple(wire_params)
computed_digest = source_plan_digest(parsed)
if self.source_digest != computed_digest:
raise ValueError(
"nccl_m2n init source digest does not match params: "
f"computed {computed_digest}, declared {self.source_digest}"
)
return parsed
@property
def nccl_unique_id_bytes(self) -> bytes | None:
if self.nccl_unique_id_b64 is None:
return None
return decode_nccl_unique_id(
master_address=None,
master_port=None,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="M2NWeightTransferInitInfo data plane",
)
@dataclass
class M2NWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Per-round update info: which parameters this chunk carries, in order.
Shapes, dtypes and layouts were fixed at init, so a round only needs to say
what is coming and in what order — both sides must issue their reshards in
exactly that order.
"""
names: list[str]
# ---------------------------------------------------------------------------
# Worker engine
# ---------------------------------------------------------------------------
class M2NWeightTransferEngine(
WeightTransferEngine[M2NWeightTransferInitInfo, M2NWeightTransferUpdateInfo]
):
"""Inference-side engine: receives each parameter with one reshard.
Resolvable parameters are resharded from the trainer layout directly into
each worker's live local model storage. Unresolvable parameters receive a
full tensor and use `load_weights`. In both cases, the trainer sends its
local shards without materializing a full tensor.
"""
init_info_cls = M2NWeightTransferInitInfo
update_info_cls = M2NWeightTransferUpdateInfo
# The destination plan is resolved against the model this engine was built
# for, so it cannot be retargeted at a draft model mid-flight.
supports_draft_weight_update = False
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: Any = None
self._m2n: Any = None
self._handle: Any = None
self._metas: list[M2NParamMeta] = []
self._dst_mesh: M2NMesh | None = None
self._parameter_destinations: list[M2NDestination] = []
self._index: dict[str, int] = {}
self._uses_load_weights = False
def init_transfer_engine(self, init_info: M2NWeightTransferInitInfo) -> None:
"""Agree on source and local readiness before constructing NCCL."""
if self._handle is not None or self.model_update_group is not None:
raise RuntimeError(
"nccl_m2n weight transfer is already initialized; call "
"shutdown_weight_transfer_engine() before initializing again"
)
rendezvous_info = NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
rank_offset=init_info.rank_offset,
world_size=init_info.world_size,
)
metadata_group = worker_init_metadata_group(
rendezvous_info, self.parallel_config
)
wire_params: tuple[M2NWireParam, ...] = ()
source_error: str | None = None
try:
self._m2n = import_m2n()
wire_params = init_info.parse_wire_params()
self._prepare_source_plan(wire_params, init_info.rank_offset)
except Exception as exc:
source_error = f"{type(exc).__name__}: {exc}"
self._metas = []
check_source_plan_agreement(
metadata_group,
wire_params,
expected_digest=init_info.source_digest,
local_error=source_error,
)
destination_error: str | None = None
try:
self._prepare_destination_plan(init_info)
except Exception as exc:
destination_error = f"{type(exc).__name__}: {exc}"
self._parameter_destinations = []
unique_id_bytes: bytes | None = None
data_plane_error: str | None = None
try:
unique_id_bytes = init_info.nccl_unique_id_bytes
except Exception as exc:
data_plane_error = f"{type(exc).__name__}: {exc}"
check_data_plane_agreement(
metadata_group,
unique_id_bytes,
data_plane_error,
)
handle = None
group = None
try:
runtime_error = destination_error
nccl = None
device = None
if runtime_error is None:
try:
nccl, device, handle = prepare_m2n_local_runtime(
self._m2n, init_info.max_cta
)
except Exception as exc:
runtime_error = f"{type(exc).__name__}: {exc}"
check_runtime_ready_agreement(metadata_group, runtime_error)
assert nccl is not None and device is not None and handle is not None
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
# Keep the merged destination-placement publication unchanged: source
# agreement is independent of worker-side destination selection.
mine = [
destination.placements for destination in self._parameter_destinations
]
agreed = publish_destination_placements(
group,
self.device,
torch.cuda.current_stream(),
init_info.rank_offset,
mine,
len(self._metas),
)
complete_plan_error: str | None = None
try:
if agreed != mine:
mismatched = next(
meta.name
for meta, ours, theirs in zip(self._metas, mine, agreed)
if ours != theirs
)
raise RuntimeError(
"inference workers resolved different nccl_m2n "
f"destinations (first mismatch: '{mismatched}'); all "
"workers must run the same model and parallel config"
)
assert self._dst_mesh is not None
validate_transfer_plan(self._metas, self._dst_mesh, agreed)
except Exception as exc:
complete_plan_error = f"{type(exc).__name__}: {exc}"
assert self._dst_mesh is not None
check_complete_plan_agreement(
metadata_group,
init_info.source_digest,
self._dst_mesh,
agreed,
complete_plan_error,
)
except BaseException:
if group is not None:
try:
# Transport cleanup only; reload validity belongs to the
# model-update lifecycle, not this backend.
group.abort()
except BaseException:
logger.exception(
"failed to abort partially initialized nccl_m2n communicator"
)
if handle is not None:
try:
handle.destroy()
except BaseException:
logger.exception(
"failed to destroy partially initialized nccl_m2n handle"
)
raise
self._handle = handle
self.model_update_group = group
def _prepare_source_plan(
self,
wire_params: tuple[M2NWireParam, ...],
num_trainer_ranks: int,
) -> None:
"""Validate worker-visible source metadata without constructing NCCL."""
self._metas = []
for param in wire_params:
dtype = getattr(torch, param.dtype_name, None)
if not isinstance(dtype, torch.dtype):
raise ValueError(
f"parameter '{param.name}' has invalid dtype name "
f"'{param.dtype_name}'; expected the name of a torch.dtype"
)
check_transferable(param.name, dtype, param.shape)
layout = M2NLayout(M2NMesh(param.src_mesh_dims, 0), param.src_placements)
if layout.mesh.size != num_trainer_ranks:
raise ValueError(
f"parameter '{param.name}' source mesh covers "
f"{layout.mesh.size} ranks, but there are "
f"{num_trainer_ranks} trainer ranks"
)
meta = M2NParamMeta(param.name, dtype, param.shape, layout)
expected_local_shape(meta)
self._metas.append(meta)
def _prepare_destination_plan(self, init_info: M2NWeightTransferInitInfo) -> None:
"""Resolve the merged-#51520 in-place/full-fallback destination plan."""
# The inference topology, as declared by the trainer. This is the mesh
# a sharded destination is placed over; a replicated one is described
# by the alternate descriptor `resolve_layout` derives from it, which
# covers the same ranks but is not this mesh.
self._dst_mesh = M2NMesh(
cast(tuple[int, int], tuple(init_info.dst_mesh_dims)),
init_info.rank_offset,
)
parallel_config = self.parallel_config
shard_axis_size = self._dst_mesh.dims[DESTINATION_SHARD_AXIS]
quantization = getattr(self.model_config, "quantization", None)
# A quantized weight loader may dequantize / transpose / requantize, and
# under PP a rank's local shape is not the checkpoint shape split over
# the shard axis. Both break shape-derived resolution, so take the
# fallback.
allow_direct = (
quantization is None and parallel_config.pipeline_parallel_size == 1
)
# Match every incoming checkpoint parameter to its rank-local target.
# Resolvable parameters can be resharded directly into live model
# storage; the rest receive a full tensor for `load_weights`.
self._parameter_destinations = resolve_parameter_destinations(
self.model,
[m.name for m in self._metas],
[m.dtype for m in self._metas],
[m.shape for m in self._metas],
num_workers=self._dst_mesh.size,
shard_axis_size=shard_axis_size,
allow_direct=allow_direct,
)
validate_transfer_plan(
self._metas,
self._dst_mesh,
[destination.placements for destination in self._parameter_destinations],
)
# Update requests carry names only, so cache their plan indices. Track
# whether any fallback entry requires the `load_weights` lifecycle.
self._index = {meta.name: i for i, meta in enumerate(self._metas)}
self._uses_load_weights = any(
not destination.direct for destination in self._parameter_destinations
)
def start_weight_update(self) -> None:
"""Set up layerwise reloading, but only if some parameter needs it.
Directly-resharded parameters are written in place and never go through
`load_weights`, so a plan with no fallback entries has nothing to
reload.
"""
if not self._uses_load_weights:
return
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading when the plan uses fallback entries."""
if not self._uses_load_weights:
return
from vllm.model_executor.model_loader.reload import finalize_layerwise_reload
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: M2NWeightTransferUpdateInfo) -> None:
"""Receive each requested parameter using its initialization-time plan."""
if self._handle is None or self.model_update_group is None:
raise RuntimeError(
"nccl_m2n weight transfer not initialized. "
"Call init_transfer_engine() first."
)
if not isinstance(update_info.names, list) or any(
not isinstance(name, str) for name in update_info.names
):
raise TypeError("nccl_m2n update names must be a list of strings")
requested = []
for name in update_info.names:
index = self._index.get(name)
if index is None:
raise ValueError(
f"parameter '{name}' was not declared at init; the "
"trainer must send the same parameter set it announced"
)
requested.append((name, index))
from vllm.model_executor.model_loader.mtp_validation import (
disable_mtp_completeness_check,
)
stream = torch.cuda.current_stream()
# Reshard lazily so one `load_weights` invocation sees the complete
# fallback sequence without staging every full tensor at once. Direct
# destinations still execute in request order as the iterator advances.
def reshard_requested_weights() -> Iterator[tuple[str, torch.Tensor]]:
for name, index in requested:
meta = self._metas[index]
destination = self._parameter_destinations[index]
buffer = destination.tensor
if buffer is None:
buffer = torch.empty(
meta.shape, dtype=meta.dtype, device=self.device
)
self._reshard(
self.model_update_group,
stream,
meta,
destination.placements,
buffer,
)
if destination.tensor is None:
# `load_weights` reads on the host stream, so the transfer
# has to have landed before it runs.
stream.synchronize()
yield name, buffer
has_fallback = any(
self._parameter_destinations[index].tensor is None for _, index in requested
)
with disable_mtp_completeness_check():
received_weights = reshard_requested_weights()
if has_fallback:
# Some model loaders keep invocation-local state to combine
# related checkpoint tensors, so preserve one iterable scope.
self.model.load_weights(received_weights)
else:
# The iterator performs the reshards. With nothing to yield to
# `load_weights`, exhaust it here to execute direct transfers.
for _ in received_weights:
pass
def _reshard(
self,
comm: Any,
stream: torch.cuda.Stream,
src_meta: M2NParamMeta,
dst_placements: Placements | None,
dst_buffer: torch.Tensor,
) -> None:
"""Reshard one parameter from its trainer layout into a worker buffer.
`dst_placements` describes the buffer's logical placement over the
worker mesh; `dst_buffer` is this rank's physical storage.
"""
assert self._dst_mesh is not None
m2n = self._m2n
src_mesh, resolved_src_placements = resolve_layout(
src_meta.source_layout.mesh, src_meta.source_layout.placements
)
dst_mesh, resolved_dst_placements = resolve_layout(
self._dst_mesh, dst_placements
)
m2n.reshard(
None,
dst_buffer,
comm,
stream,
src_mesh=to_mesh(m2n, src_mesh),
src_placements=to_placements(m2n, resolved_src_placements),
dst_mesh=to_mesh(m2n, dst_mesh),
dst_placements=to_placements(m2n, resolved_dst_placements),
handle=self._handle,
)
def shutdown(self) -> None:
"""Finish pending GPU work and release M2N communication state."""
handle, self._handle = self._handle, None
group, self.model_update_group = self.model_update_group, None
self._m2n = None
self._metas = []
self._dst_mesh = None
self._parameter_destinations = []
self._index = {}
self._uses_load_weights = False
first_error: BaseException | None = None
def attempt(action: Any, message: str) -> None:
nonlocal first_error
try:
action()
except BaseException as exc:
if first_error is None:
first_error = exc
logger.exception(message)
if handle is not None or group is not None:
# M2N does not synchronize caller streams on finalize.
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
if handle is not None:
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
if group is not None:
attempt(group.abort, "failed to abort nccl_m2n communicator")
if first_error is not None:
raise first_error
@@ -1,261 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Destination-layout resolution for the NCCL M2N weight transfer backend.
For every incoming checkpoint parameter the worker needs a destination buffer
plus its placement over the inference mesh. Two outcomes are possible:
* **direct** — when the model is not quantized, the parameter maps 1:1 onto a
live vLLM parameter and its loader is a small, known-safe copy or tensor-
parallel loader. The loader's declared input/output dimension determines the
placement; shapes are validation, not an inference mechanism. The reshard
writes straight into the live parameter, so each rank receives only its own
shard and nothing is copied afterwards.
* **fallback** — anything else. The reshard delivers the whole tensor to every
rank and `load_weights` does the sharding, exactly as the broadcast NCCL
backend does. Fused parameters (`qkv_proj`, `gate_up_proj`, MoE `w13`/`w2`)
take this path: the checkpoint name does not name a vLLM parameter, so there
is nothing to resolve against.
Correctness never depends on a parameter resolving — the fallback is always
available and is the same path the existing backend uses.
"""
import inspect
import math
from collections.abc import Sequence
from dataclasses import dataclass
from functools import partial
from typing import cast
import torch
from vllm.distributed.weight_transfer.m2n_common import (
DESTINATION_REPLICA_AXIS,
DESTINATION_SHARD_AXIS,
MESH_NDIMS,
REPLICATE,
REPLICATED,
Placements,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
_DIRECT_WEIGHT_LOADING_BLOCKLIST = {
"vllm.model_executor.models.gpt2.GPT2Model",
}
def _module_type_name(module: torch.nn.Module) -> str:
cls = type(module)
return f"{cls.__module__}.{cls.__name__}"
def _destination_placements(shard_dim: int) -> Placements:
"""Place one tensor shard on the configured destination mesh axis."""
placements = [REPLICATE] * MESH_NDIMS
placements[DESTINATION_REPLICA_AXIS] = REPLICATE
placements[DESTINATION_SHARD_AXIS] = shard_dim
return cast(Placements, tuple(placements))
@dataclass
class M2NDestination:
"""Where one checkpoint parameter lands on this worker."""
name: str
placements: Placements | None
"""Placement over the inference mesh, or `REPLICATED` for the fallback."""
tensor: torch.Tensor | None
"""The live parameter view to reshard into, or None for the fallback path
(the engine allocates a full replica per round and calls `load_weights`)."""
@property
def direct(self) -> bool:
return self.tensor is not None
def _base_loader_shard_dim(
param: torch.nn.Parameter,
shard_axis_size: int,
) -> int | None:
"""Return a known-safe loader's declared shard dim, or reject it.
Loader identity is deliberately fail-closed. A missing loader is not
treated as ``default_weight_loader`` because a model-level ``load_weights``
implementation may transform the tensor before it reaches that default.
Wrappers and partials are also rejected: recognizing their wrapped callable
would ignore behavior added by the wrapper.
"""
loader = getattr(param, "weight_loader", None)
if loader is None or isinstance(loader, partial):
return None
owner = loader.__self__ if inspect.ismethod(loader) else None
loader_fn = loader.__func__ if inspect.ismethod(loader) else loader
# Lazy imports avoid making the weight-transfer registry import all model
# executor layers during normal vLLM startup.
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
replicated_loaders = {
default_weight_loader,
ReplicatedLinear.weight_loader,
}
column_loaders = {
ColumnParallelLinear.weight_loader,
ColumnParallelLinear.weight_loader_v2,
}
row_loaders = {
RowParallelLinear.weight_loader,
RowParallelLinear.weight_loader_v2,
}
# These attributes signal packing, a transform, or pre-sharded checkpoint
# storage. Such semantics cannot be reproduced by a plain M2N placement.
# Check them before accepting replicated loaders too: model-level loading
# may transform a tensor before handing it to an otherwise plain copier.
metadata_attrs = ("packed_dim", "packed_factor", "marlin_tile_size")
flag_attrs = ("needs_scalar_to_array", "is_sharded_weight", "is_transposed")
if any(getattr(param, attr, None) is not None for attr in metadata_attrs):
return None
if any(bool(getattr(param, attr, False)) for attr in flag_attrs):
return None
if loader_fn in replicated_loaders:
return REPLICATE
if loader_fn not in column_loaders | row_loaders:
return None
if owner is None or getattr(owner, "tp_size", None) != shard_axis_size:
return None
attr = "output_dim" if loader_fn in column_loaders else "input_dim"
dim = getattr(param, attr, None)
return dim if isinstance(dim, int) and dim >= 0 else None
def _validated_shard_dim(
global_shape: Sequence[int],
local_shape: Sequence[int],
declared_dim: int,
shard_axis_size: int,
) -> int | None:
"""Validate a loader-declared shard dimension against both shapes."""
if len(global_shape) != len(local_shape):
return None
if declared_dim == REPLICATE:
return REPLICATE if tuple(global_shape) == tuple(local_shape) else None
if declared_dim >= len(global_shape):
return None
for dim, (whole, local) in enumerate(zip(global_shape, local_shape)):
expected = local * shard_axis_size if dim == declared_dim else local
if whole != expected:
return None
return declared_dim
def resolve_parameter_destinations(
model: torch.nn.Module,
names: Sequence[str],
dtypes: Sequence[torch.dtype],
shapes: Sequence[Sequence[int]],
*,
num_workers: int,
shard_axis_size: int,
allow_direct: bool,
) -> list[M2NDestination]:
"""Build this worker's destination plan, one entry per checkpoint parameter.
Placements are relative to the inference mesh: axis 0 replicates and axis 1
shards. `allow_direct=False` forces every parameter onto the fallback path;
the engine sets it for pipeline-parallel or quantized deployments, where a
parameter's local shape is not simply the checkpoint shape split across the
shard axis. Parameters below a model type with known model-level checkpoint
preprocessing also fall back, since bypassing `load_weights` would skip the
transformation.
"""
params = dict(model.named_parameters()) if allow_direct else {}
blocked_param_ids = {
id(param)
for module in model.modules()
if _module_type_name(module) in _DIRECT_WEIGHT_LOADING_BLOCKLIST
for param in module.parameters()
}
destinations: list[M2NDestination] = []
for name, dtype, shape in zip(names, dtypes, shapes):
param = params.get(name)
dim = None
if (
param is not None
and id(param) not in blocked_param_ids
and param.dtype == dtype
and param.data.is_contiguous()
and num_workers % shard_axis_size == 0
):
declared_dim = _base_loader_shard_dim(param, shard_axis_size)
if declared_dim is not None:
dim = _validated_shard_dim(
shape, param.shape, declared_dim, shard_axis_size
)
if dim is None:
# A replicated parameter is identical on every rank, so it needs no
# placement of its own — REPLICATED lets resolve_layout spread it
# over all the workers regardless of how the mesh is factored.
destinations.append(M2NDestination(name, REPLICATED, None))
elif dim == REPLICATE:
assert param is not None
destinations.append(M2NDestination(name, REPLICATED, param.data))
else:
assert param is not None
destinations.append(
M2NDestination(name, _destination_placements(dim), param.data)
)
# Layerwise reload finalizes a module as a unit. Mixing direct and fallback
# parameters within one module can copy stale storage over the direct load.
fallback_modules = {
name.rpartition(".")[0]
for name, destination in zip(names, destinations)
if not destination.direct and name in params
}
destinations = [
M2NDestination(destination.name, REPLICATED, None)
if destination.direct
and destination.name.rpartition(".")[0] in fallback_modules
else destination
for destination in destinations
]
num_direct = sum(d.direct for d in destinations)
parameter_bytes = [
math.prod(shape) * dtype.itemsize for dtype, shape in zip(dtypes, shapes)
]
total_bytes = sum(parameter_bytes)
direct_bytes = sum(
nbytes
for destination, nbytes in zip(destinations, parameter_bytes)
if destination.direct
)
direct_byte_percentage = 100 * direct_bytes / total_bytes if total_bytes else 0
logger.info(
"nccl_m2n destination plan: %d/%d parameters resharded directly into "
"the model, %d via full-tensor fallback; direct byte coverage: "
"%d/%d bytes (%.1f%%)",
num_direct,
len(destinations),
len(destinations) - num_direct,
direct_bytes,
total_bytes,
direct_byte_percentage,
)
return destinations
@@ -1,207 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Trainer-side weight sources for the NCCL M2N backend.
m2n plans a transfer from both sides' layouts, so the trainer has to say how it
holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not
express. This module supplies the m2n flavor of both, and the DTensor-backed
implementation that covers the common trainer. Each parameter owns its source
mesh and placements because dense and expert weights can use different
factorizations over the same trainer ranks.
"""
from collections.abc import Callable, Iterator, Sequence
from dataclasses import dataclass
from typing import Any
import torch
from vllm.distributed.weight_transfer.base import WeightSource
from vllm.distributed.weight_transfer.m2n_common import (
REPLICATE,
REPLICATED,
M2NLayout,
M2NMesh,
M2NParamMeta,
Placements,
)
__all__ = [
"M2NWeightSource",
"M2NManifestEntry",
"ManifestM2NWeightSource",
"DTensorModuleSource",
"mesh_from_tensor",
"placements_from_tensor",
]
class M2NWeightSource(WeightSource):
"""A `WeightSource` that also describes how the trainer holds its weights.
Unlike `ModuleSource`, iteration yields each rank's **local shard**, not a
materialized full tensor: gathering is exactly the cost m2n removes.
"""
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Name, dtype, full shape, and source layout for each parameter."""
raise NotImplementedError
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
"""Yield `(name, local buffer)` pairs in metadata order.
Providers must enqueue materialization on the caller's current CUDA
stream, or return a buffer whose writes are already visible to it.
Buffers may be torch tensors, CUDA Array Interface objects, or raw
device pointers accepted by nccl4py. Providers own correct
communicator-device placement. Raw-pointer providers additionally own
allocation lifetime, dtype, and shape correctness.
"""
raise NotImplementedError
@dataclass(frozen=True)
class M2NManifestEntry:
"""One immutable source descriptor with a lazy rank-local tensor view."""
key: str
dtype: torch.dtype
global_shape: tuple[int, ...]
source_layout: M2NLayout
local_tensor: Callable[[], Any]
def __post_init__(self) -> None:
object.__setattr__(self, "global_shape", tuple(self.global_shape))
if not self.key:
raise ValueError("M2N manifest key must not be empty")
if not isinstance(self.dtype, torch.dtype):
raise TypeError(
f"manifest entry '{self.key}' has invalid dtype {self.dtype!r}"
)
if not isinstance(self.source_layout, M2NLayout):
raise TypeError(
f"manifest entry '{self.key}' source_layout must be an M2NLayout"
)
if not callable(self.local_tensor):
raise TypeError(
f"manifest entry '{self.key}' local_tensor must be callable"
)
@property
def metadata(self) -> M2NParamMeta:
"""Return the serializable portion consumed during initialization."""
return M2NParamMeta(
self.key,
self.dtype,
self.global_shape,
self.source_layout,
)
class ManifestM2NWeightSource(M2NWeightSource):
"""An ordered M2N source backed by lazy, rank-local manifest entries."""
def __init__(self, entries: Sequence[M2NManifestEntry]) -> None:
self._entries = tuple(entries)
self._metadata = tuple(entry.metadata for entry in self._entries)
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Return a fresh list while preserving the immutable manifest order."""
return list(self._metadata)
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
"""Materialize one rank-local buffer at a time."""
for entry in self._entries:
yield entry.key, entry.local_tensor()
def _placement_code(placement: Any) -> int:
"""Map a `torch.distributed` placement onto an m2n placement code."""
name = type(placement).__name__
if name == "Replicate":
return REPLICATE
if name == "Shard":
return int(placement.dim)
raise ValueError(
f"nccl_m2n cannot express the {name} placement; only Replicate and "
"Shard are supported"
)
def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh:
"""The mesh a parameter lives on, as an `M2NMesh`.
Tensors without DeviceMesh metadata are treated as replicated. Implicitly
sharded tensors require a custom source with an explicit layout.
"""
device_mesh = getattr(tensor, "device_mesh", None)
if device_mesh is None:
return M2NMesh((num_trainer_ranks, 1), 0)
grid = device_mesh.mesh
ranks = grid.flatten().tolist()
if ranks != list(range(len(ranks))):
raise ValueError(
"nccl_m2n requires the trainer's device mesh to cover the "
f"contiguous rank interval [0, {len(ranks)}); got {ranks}"
)
if grid.ndim > 2:
raise ValueError(
f"nccl_m2n supports 1-D and 2-D device meshes, got {grid.ndim}-D"
)
dims = tuple(grid.shape)
return M2NMesh(dims if len(dims) == 2 else (dims[0], 1), 0)
def placements_from_tensor(tensor: torch.Tensor) -> Placements | None:
"""How a parameter is placed over its mesh, or `REPLICATED`.
`REPLICATED` is returned rather than a `(REPLICATE, REPLICATE)` pair: m2n
cannot express that, and the size-1-axis encoding it needs instead is
applied later by `resolve_layout`, which owns that workaround.
"""
placements = getattr(tensor, "placements", None)
if placements is None:
return REPLICATED
codes = [_placement_code(p) for p in placements]
if all(code == REPLICATE for code in codes):
return REPLICATED
if len(codes) == 1:
return (codes[0], REPLICATE)
return (codes[0], codes[1])
class DTensorModuleSource(M2NWeightSource):
"""`M2NWeightSource` over `module.named_parameters()`.
Covers both the FSDP/DTensor trainer (placement read off each parameter,
local shard yielded via `to_local()`) and the plain replicated trainer, with
no special casing. Trainers with a custom producer (a Megatron export, MoE
re-fusing) subclass `M2NWeightSource` instead.
"""
def __init__(self, module: torch.nn.Module, num_trainer_ranks: int) -> None:
"""Wrap `module`; `num_trainer_ranks` sizes the mesh for plain tensors."""
self._module = module
self._num_trainer_ranks = num_trainer_ranks
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Read each global shape and source layout without gathering shards."""
return [
M2NParamMeta(
name,
p.dtype,
tuple(p.shape),
M2NLayout(
mesh_from_tensor(p, self._num_trainer_ranks),
placements_from_tensor(p),
),
)
for name, p in self._module.named_parameters()
]
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
"""Yield each parameter's local shard (`to_local()`), or the tensor itself."""
for name, param in self._module.named_parameters():
to_local = getattr(param, "to_local", None)
yield name, (to_local() if callable(to_local) else param)
@@ -1,628 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Trainer-side weight transfer engine for the NCCL M2N backend.
Symmetric to `M2NWeightTransferEngine` but in the training process. Every
trainer rank joins the shared communicator and runs every reshard, sending its
own local shard; only rank 0 touches the inference control plane.
Workers join the communicator during init and run reshard from inside
`update_weights`, so those two RPCs overlap with work on this side. Both run on
a helper thread and are joined afterwards, which keeps that overlap inside the
engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather than
in every caller.
"""
import socket
from collections.abc import Mapping
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
import torch
from vllm.distributed.weight_transfer.base import (
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
)
from vllm.distributed.weight_transfer.m2n_common import (
M2N_WIRE_SCHEMA_VERSION,
M2NMesh,
M2NParamMeta,
M2NWireParam,
Placements,
check_complete_plan_agreement,
check_data_plane_agreement,
check_runtime_ready_agreement,
check_source_plan_agreement,
check_transferable,
create_m2n_communicator,
expected_local_shape,
import_m2n,
prepare_m2n_local_runtime,
publish_destination_placements,
resolve_layout,
source_plan_digest,
to_mesh,
to_placements,
validate_local_buffer,
validate_transfer_plan,
)
from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource
from vllm.distributed.weight_transfer.nccl_common import (
decode_nccl_unique_id,
stateless_init_metadata_group,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.distributed.utils import StatelessProcessGroup
logger = init_logger(__name__)
__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"]
@dataclass(kw_only=True)
class M2NTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for nccl_m2n.
`rank` (from `TrainerInitInfo`) is this trainer process's rank; it is also
its rank in the shared communicator, since the trainer occupies
`[0, num_trainer_ranks)`. Rank 0 drives the control plane.
"""
backend: ClassVar[str] = "nccl_m2n"
master_address: str
master_port: int
world_size: int
"""Trainer ranks + all inference workers."""
num_trainer_ranks: int = 1
dst_mesh_dims: tuple[int, int] | None = None
"""How the inference ranks are laid out: axis 0 replicates and axis 1
shards. The trainer declares it so both sides describe the destination
identically; defaults to a flat `(num_workers, 1)`, which is all a
replicated destination needs.
Destination meshes remain common to every parameter; per-parameter
destination meshes are intentionally future work.
"""
nccl_unique_id_b64: str | None = field(default=None, repr=False)
"""Optional UID for the NCCL data plane; TCP remains the metadata plane."""
max_cta: int | None = None
listen_socket: socket.socket | None = field(default=None, repr=False, compare=False)
"""Optional rank-zero TCPStore reservation; never sent to workers."""
@property
def destination_mesh_dims(self) -> tuple[int, int]:
"""`dst_mesh_dims`, or a flat mesh over every inference worker."""
if self.dst_mesh_dims is not None:
return cast(tuple[int, int], tuple(self.dst_mesh_dims))
return (self.world_size - self.num_trainer_ranks, 1)
def __post_init__(self) -> None:
"""Reject rank counts or destination meshes that cannot form a group."""
if not isinstance(self.master_address, str) or not self.master_address:
raise ValueError("`master_address` must name the metadata rendezvous")
if (
not isinstance(self.master_port, int)
or isinstance(self.master_port, bool)
or not 0 < self.master_port < 65536
):
raise ValueError("`master_port` must be a valid metadata port")
if not 0 < self.num_trainer_ranks < self.world_size:
raise ValueError(
f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at "
f"least one worker in world_size {self.world_size}"
)
num_workers = self.world_size - self.num_trainer_ranks
if self.dst_mesh_dims is not None:
dims = tuple(self.dst_mesh_dims)
try:
mesh = M2NMesh(cast(tuple[int, int], dims), self.num_trainer_ranks)
except (TypeError, ValueError) as exc:
raise ValueError(f"invalid `dst_mesh_dims` {dims}: {exc}") from exc
if mesh.size != num_workers:
raise ValueError(
f"`dst_mesh_dims` {dims} must be two dims covering the "
f"{num_workers} inference workers"
)
if not 0 <= self.rank < self.num_trainer_ranks:
raise ValueError(
f"trainer `rank` ({self.rank}) must be below "
f"`num_trainer_ranks` ({self.num_trainer_ranks})"
)
if self.listen_socket is not None:
if self.rank != 0:
raise ValueError("only trainer rank zero may reserve the TCPStore")
if not isinstance(self.listen_socket, socket.socket):
raise TypeError("listen_socket must be a socket.socket")
if self.listen_socket.fileno() < 0:
raise ValueError("listen_socket is already closed")
@property
def nccl_unique_id_bytes(self) -> bytes | None:
if self.nccl_unique_id_b64 is None:
return None
return decode_nccl_unique_id(
master_address=None,
master_port=None,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="M2NTrainerInitInfo data plane",
)
class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitInfo]):
"""Trainer-side engine: sends every rank's local shard, once per parameter.
Called on every trainer rank. All ranks join the communicator and run every
reshard; only rank 0 touches the control plane.
"""
init_info_cls = M2NTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: M2NWeightSource,
is_sender: bool,
) -> None:
"""Hold the client and source; `trainer_init` fills in the group."""
super().__init__(client=client, source=source, is_sender=is_sender)
self.group: Any = None
self._m2n: Any = None
self._handle: Any = None
self._metas: list[M2NParamMeta] = []
self._wire_params: list[M2NWireParam] = []
self._source_digest = ""
self._dst_mesh: M2NMesh | None = None
self._dst_placements: list[Placements | None] = []
self._metadata_group: StatelessProcessGroup | None = None
self._num_trainer_ranks = 0
@classmethod
def trainer_init(
cls,
init_info: M2NTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
"""Build the engine and rendezvous with the inference side.
Runs on *every* trainer rank. Rank 0 additionally ships the transfer
plan to the workers and drives the control plane; the other ranks only
build local state and join the communicator. Every rank participates
in every reshard and sends its local shard.
Build a local validation envelope, start the worker RPC without
waiting, join the metadata rendezvous, and require unanimous source,
data-plane, and runtime readiness before constructing NCCL.
"""
engine = cls(
client=client,
source=cast(M2NWeightSource, source),
is_sender=init_info.is_sender,
)
engine._initialize(init_info, source)
return engine
def _initialize(
self,
init_info: M2NTrainerInitInfo,
source: WeightSource | None,
) -> None:
"""Run source validation, metadata agreement, and NCCL rendezvous."""
self._num_trainer_ranks = init_info.num_trainer_ranks
local_source_error: str | None = None
try:
if not isinstance(source, M2NWeightSource):
raise TypeError(
"nccl_m2n needs per-parameter layouts, so its source must "
f"be an M2NWeightSource; got {type(source).__name__}"
)
self._m2n = import_m2n()
self._prepare_source_plan(source, init_info.num_trainer_ranks)
except Exception as exc:
local_source_error = f"{type(exc).__name__}: {exc}"
self._metas = []
self._wire_params = []
self._source_digest = source_plan_digest([])
executor: ThreadPoolExecutor | None = None
init_future: Future[None] | None = None
if self.is_sender:
executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="m2n-weight-sync"
)
init_future = executor.submit(
self.client.init_weight_transfer_engine,
self._worker_init_info(init_info),
)
handle = None
group = None
try:
reservation = init_info.listen_socket
try:
metadata_group = stateless_init_metadata_group(
init_info.master_address,
init_info.master_port,
init_info.rank,
init_info.world_size,
listen_socket=reservation,
)
finally:
init_info.listen_socket = None
if reservation is not None:
reservation.close()
plan_digest = check_source_plan_agreement(
metadata_group,
self._wire_params,
expected_digest=self._source_digest,
local_error=local_source_error,
)
# Retain the common destination mesh and the merged NCCL publication
# of per-parameter placements.
self._dst_mesh = M2NMesh(
init_info.destination_mesh_dims, init_info.num_trainer_ranks
)
unique_id_bytes: bytes | None = None
data_plane_error: str | None = None
try:
unique_id_bytes = init_info.nccl_unique_id_bytes
except Exception as exc:
data_plane_error = f"{type(exc).__name__}: {exc}"
check_data_plane_agreement(
metadata_group,
unique_id_bytes,
data_plane_error,
)
runtime_error: str | None = None
nccl = None
device = None
try:
nccl, device, handle = prepare_m2n_local_runtime(
self._m2n, init_info.max_cta
)
except Exception as exc:
runtime_error = f"{type(exc).__name__}: {exc}"
check_runtime_ready_agreement(metadata_group, runtime_error)
assert nccl is not None and device is not None and handle is not None
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
destination_placements = self._receive_destination_plan(
group,
init_info.num_trainer_ranks,
torch.device("cuda", device),
)
complete_plan_error: str | None = None
try:
assert self._dst_mesh is not None
validate_transfer_plan(
self._metas, self._dst_mesh, destination_placements
)
except Exception as exc:
complete_plan_error = f"{type(exc).__name__}: {exc}"
assert self._dst_mesh is not None
check_complete_plan_agreement(
metadata_group,
plan_digest,
self._dst_mesh,
destination_placements,
complete_plan_error,
)
if init_future is not None:
init_future.result()
logger.info(
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
"inference ranks [%d, %d)",
len(self._metas),
init_info.num_trainer_ranks,
init_info.num_trainer_ranks,
init_info.world_size,
)
logger.info("nccl_m2n source plan digest: %s", plan_digest)
except BaseException:
if group is not None:
try:
# Transport cleanup only; reload validity belongs to the
# model-update lifecycle, not this backend.
group.abort()
except BaseException:
logger.exception(
"failed to abort partially initialized nccl_m2n communicator"
)
if handle is not None:
try:
handle.destroy()
except BaseException:
logger.exception(
"failed to destroy partially initialized nccl_m2n handle"
)
raise
finally:
if executor is not None:
executor.shutdown(wait=False, cancel_futures=True)
self._metadata_group = metadata_group
self._handle = handle
self.group = group
self._dst_placements = destination_placements
def _receive_destination_plan(
self, group: Any, first_worker_rank: int, device: torch.device
) -> list[Placements | None]:
"""Receive the inference-side per-parameter placements."""
# The first inference worker publishes one placement per parameter.
# Every trainer rank receives the complete plan in this single,
# initialization-time broadcast.
placements = publish_destination_placements(
group,
device,
torch.cuda.current_stream(),
first_worker_rank,
None,
len(self._metas),
)
return placements
def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]:
"""Handshake payload: rendezvous, both meshes, and the transfer plan."""
payload: dict[str, Any] = {
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"master_address": init_info.master_address,
"master_port": init_info.master_port,
"nccl_unique_id_b64": init_info.nccl_unique_id_b64,
"rank_offset": init_info.num_trainer_ranks,
"world_size": init_info.world_size,
"dst_mesh_dims": list(init_info.destination_mesh_dims),
"source_digest": self._source_digest,
"params": [param.to_dict() for param in self._wire_params],
"max_cta": init_info.max_cta,
}
return {key: value for key, value in payload.items() if value is not None}
def _prepare_source_plan(
self, source: M2NWeightSource, num_trainer_ranks: int
) -> None:
"""Build and validate this rank's immutable source plan."""
self._metas = source.metadata()
names: set[str] = set()
for meta in self._metas:
if not isinstance(meta, M2NParamMeta):
raise TypeError(
"nccl_m2n source metadata must contain M2NParamMeta "
f"entries, got {type(meta).__name__}"
)
if meta.name in names:
raise ValueError(
"nccl_m2n source metadata contains duplicate parameter "
f"'{meta.name}'"
)
names.add(meta.name)
check_transferable(meta.name, meta.dtype, meta.shape)
layout = meta.source_layout
if layout.mesh.start_rank != 0:
raise ValueError(
f"parameter '{meta.name}' source mesh must start at rank 0, "
f"got {layout.mesh.start_rank}"
)
if layout.mesh.size != num_trainer_ranks:
raise ValueError(
f"parameter '{meta.name}' source mesh covers "
f"{layout.mesh.size} ranks, but there are "
f"{num_trainer_ranks} trainer ranks"
)
expected_local_shape(meta)
self._wire_params = [
M2NWireParam(
name=meta.name,
dtype_name=str(meta.dtype).split(".")[-1],
shape=meta.shape,
src_mesh_dims=meta.source_layout.mesh.dims,
src_placements=meta.source_layout.placements,
)
for meta in self._metas
]
self._source_digest = source_plan_digest(self._wire_params)
def send_weights(self) -> None:
"""Drive one update round: start, reshard concurrently, then finish."""
if self._handle is None or self.group is None:
raise RuntimeError("nccl_m2n trainer engine is not initialized")
if not self.is_sender:
try:
self._send()
except BaseException:
try:
self._receive_round_outcome()
except BaseException:
logger.exception(
"failed to consume nccl_m2n round outcome after local failure"
)
raise
self._receive_round_outcome()
return
try:
self.client.start_weight_update()
executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="m2n-weight-sync"
)
try:
update = executor.submit(
self.client.update_weights,
{"names": [m.name for m in self._metas]},
)
# Surface an immediate worker-side rejection before trainers
# enter the collective sequence.
if update.done():
update.result()
self._send()
update.result()
finally:
executor.shutdown(wait=False, cancel_futures=True)
self.client.finish_weight_update()
except BaseException as exc:
try:
self._publish_round_outcome(exc)
except BaseException:
logger.exception("failed to publish nccl_m2n round failure")
raise
self._publish_round_outcome(None)
def _publish_round_outcome(self, failure: BaseException | None) -> None:
"""Tell non-sender trainer ranks whether worker finalization passed."""
if self._num_trainer_ranks <= 1:
return
if self._metadata_group is None:
raise RuntimeError("nccl_m2n metadata group is unavailable")
envelope = {
"ok": failure is None,
"error": (
None if failure is None else f"{type(failure).__name__}: {failure}"
),
}
for rank in range(1, self._num_trainer_ranks):
self._metadata_group.send_obj(envelope, rank)
# Keep rank zero's TCPStore alive until every trainer consumed its
# outcome. This also keeps the per-rank message counters aligned.
expected_ack = {"phase": "round_outcome_consumed"}
for rank in range(1, self._num_trainer_ranks):
ack = self._metadata_group.recv_obj(rank)
if ack != expected_ack:
raise RuntimeError(
f"trainer rank {rank} sent an invalid round-outcome acknowledgement"
)
def _receive_round_outcome(self) -> None:
"""Wait until trainer rank zero observed every worker's finish RPC."""
if self._metadata_group is None:
raise RuntimeError("nccl_m2n metadata group is unavailable")
outcome = self._metadata_group.recv_obj(0)
self._metadata_group.send_obj({"phase": "round_outcome_consumed"}, 0)
if not isinstance(outcome, Mapping) or set(outcome) != {"ok", "error"}:
raise RuntimeError("trainer rank zero sent an invalid round outcome")
ok = outcome["ok"]
error = outcome["error"]
if not isinstance(ok, bool) or ok != (error is None):
raise RuntimeError("trainer rank zero sent an invalid round outcome")
if not ok:
if not isinstance(error, str) or not error:
raise RuntimeError("trainer rank zero sent an invalid round error")
raise RuntimeError(
"nccl_m2n worker update failed on trainer rank zero: " + error
)
def _send(self) -> None:
"""Reshard each local shard into its worker-planned destination."""
assert self.group is not None
assert self._dst_mesh is not None
assert self.source is not None
m2n = self._m2n
stream = torch.cuda.current_stream()
device_index = torch.accelerator.current_device_index()
# Optimizer work may have used another CUDA stream. Make prior writes
# visible before a provider materializes current-stream local shards.
torch.accelerator.synchronize(device_index)
inflight_buffers: list[Any] = []
sent = 0
for index, (name, buffer) in enumerate(self.source):
if index >= len(self._metas):
raise RuntimeError(
"weight source yielded more parameters than metadata() "
f"declared (first extra parameter: '{name}')"
)
meta = self._metas[index]
if name != meta.name:
raise RuntimeError(
f"weight source yielded '{name}' at position {index}, but "
f"its metadata declared '{meta.name}'; the source must "
"iterate in a stable order"
)
validate_local_buffer(m2n, meta, buffer)
src_mesh, src_placements = resolve_layout(
meta.source_layout.mesh, meta.source_layout.placements
)
dst_mesh, dst_placements = resolve_layout(
self._dst_mesh, self._dst_placements[index]
)
raw_pointer_args = {}
if isinstance(buffer, int):
raw_pointer_args = {
"src_local_shape": expected_local_shape(meta),
"src_dtype": meta.dtype,
}
m2n.reshard(
buffer,
None,
self.group,
stream,
src_mesh=to_mesh(m2n, src_mesh),
src_placements=to_placements(m2n, src_placements),
dst_mesh=to_mesh(m2n, dst_mesh),
dst_placements=to_placements(m2n, dst_placements),
handle=self._handle,
**raw_pointer_args,
)
# Let allocator-aware buffers recycle storage in stream order.
# Other object owners must stay alive until the final synchronize;
# raw-pointer lifetime remains the source's responsibility.
record_stream = getattr(buffer, "record_stream", None)
if callable(record_stream):
record_stream(stream)
elif not isinstance(buffer, int):
inflight_buffers.append(buffer)
sent += 1
if sent != len(self._metas):
missing = self._metas[sent].name
raise RuntimeError(
f"weight source yielded {sent} parameters, but metadata() "
f"declared {len(self._metas)} (first missing: '{missing}')"
)
# As in the other weight-transfer backends, wait before returning
# rather than relying on same-stream ordering. The caller may then
# safely mutate parameters or continue work on another stream.
stream.synchronize()
def shutdown(self) -> None:
"""Finish pending GPU work and release M2N communication state."""
handle, self._handle = self._handle, None
group, self.group = self.group, None
self._metadata_group = None
self._dst_placements = []
self._dst_mesh = None
self._m2n = None
first_error: BaseException | None = None
def attempt(action: Any, message: str) -> None:
nonlocal first_error
try:
action()
except BaseException as exc:
if first_error is None:
first_error = exc
logger.exception(message)
if handle is not None or group is not None:
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
if handle is not None:
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
if group is not None:
attempt(group.abort, "failed to abort nccl_m2n communicator")
if first_error is not None:
raise first_error
@@ -1,41 +0,0 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Re-export the weight-transfer backend maintained by ModelExpress.
Install ModelExpress from the main branch's modelexpress_client/python
subdirectory.
"""
try:
from modelexpress import configure_vllm_logging
from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import (
ModelExpressWeightTransferEngine,
ModelExpressWeightTransferInitInfo,
ModelExpressWeightTransferUpdateInfo,
)
except ModuleNotFoundError as exc:
if exc.name not in {
"modelexpress",
"modelexpress_rl",
"modelexpress_rl.inference",
"modelexpress_rl.inference.engines",
"modelexpress_rl.inference.engines.vllm",
"modelexpress_rl.inference.engines.vllm.weight_transfer_engine",
}:
raise
raise ImportError(
"The 'modelexpress' weight transfer backend requires ModelExpress. "
"Install it with `uv pip install "
"'git+https://github.com/ai-dynamo/modelexpress@main"
"#subdirectory=modelexpress_client/python'`."
) from exc
configure_vllm_logging()
__all__ = [
"ModelExpressWeightTransferEngine",
"ModelExpressWeightTransferInitInfo",
"ModelExpressWeightTransferUpdateInfo",
]
@@ -1,299 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared NCCL initialization helpers for weight transfer engines.
The dense (`NCCLWeightTransferEngine`) and sparse
(`SparseNCCLWeightTransferEngine`) backends are independent engines that share
*only* their process-group initialization. That common logic lives here so the
sparse engine does not have to subclass the dense one.
"""
import socket
from dataclasses import asdict, dataclass, field
from typing import TYPE_CHECKING, Protocol
import pybase64 as base64
import torch
if TYPE_CHECKING:
from vllm.config.parallel import ParallelConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCL_UNIQUE_ID_BYTES,
)
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
)
from vllm.utils.nccl import unpinned_nccl_env
def decode_nccl_unique_id(
*,
master_address: str | None,
master_port: int | None,
nccl_unique_id_b64: str | None,
ctx: str,
) -> bytes | None:
"""Validate the rendezvous mode and decode a pre-shared unique id."""
has_uid = bool(nccl_unique_id_b64)
has_addr = master_address is not None
has_port = master_port is not None
if has_uid and (has_addr or has_port):
raise ValueError(
f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both"
)
if not has_uid:
if not (has_addr and has_port):
raise ValueError(
f"{ctx}: need nccl_unique_id_b64, or both master_address and "
f"master_port (got master_address={master_address!r}, "
f"master_port={master_port!r})"
)
return None
try:
decoded = base64.b64decode(nccl_unique_id_b64, validate=True)
except (ValueError, TypeError) as e:
raise ValueError(
f"{ctx}: nccl_unique_id_b64 is not valid standard base64 "
f"(URL-safe base64 is not accepted)"
) from e
if len(decoded) != NCCL_UNIQUE_ID_BYTES:
raise ValueError(
f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} "
f"bytes, got {len(decoded)}"
)
return decoded
@dataclass(kw_only=True)
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side initialization info for NCCL-based weight transfer backends.
Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field
means the rendezvous fields can no longer keep a fixed positional slot, so a
stale positional call fails loudly instead of silently swapping arguments.
Provide exactly one rendezvous mode:
* ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup``
rendezvous (requires torch on every rank, including the trainer), or
* ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the
128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX)
that mint the unique id out of band and share it (over HTTP, etc.). Note a
JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``.
On the unique-id path all ranks must enter init concurrently (there is no
store barrier), and every peer must honor the warm-up handshake: the
worker's communicator issues a one-element ``all_reduce`` immediately after
``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a
foreign peer must issue a matching one-element ``all_reduce`` before any
other collective or all ranks deadlock.
"""
master_address: str | None = None
master_port: int | None = None
rank_offset: int
world_size: int
nccl_unique_id_b64: str | None = field(default=None, repr=False)
packed: bool = False
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
def __post_init__(self) -> None:
_ = self.nccl_unique_id_bytes
@property
def nccl_unique_id_bytes(self) -> bytes | None:
return decode_nccl_unique_id(
master_address=self.master_address,
master_port=self.master_port,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="NCCLWeightTransferInitInfo",
)
def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
only the mode actually in use. Shared by the dense and sparse trainer
engines so the two cannot drift."""
return {key: value for key, value in asdict(init_info).items() if value is not None}
class NCCLRendezvous(Protocol):
"""The TCP rendezvous fields `trainer_init` needs.
Structural so each backend can keep its own trainer init info next to its
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
this shared module importing either.
"""
master_address: str
master_port: int
world_size: int
def stateless_init_metadata_group(
master_address: str,
master_port: int,
rank: int,
world_size: int,
*,
listen_socket: socket.socket | None = None,
) -> "StatelessProcessGroup":
"""Create only the TCPStore group used for metadata agreement."""
from vllm.distributed.utils import StatelessProcessGroup
return StatelessProcessGroup.create(
host=master_address,
port=master_port,
rank=rank,
world_size=world_size,
listen_socket=listen_socket,
)
def stateless_init_process_group(
master_address: str,
master_port: int,
rank: int,
world_size: int,
device,
) -> "PyNcclCommunicator":
"""VLLM provides `StatelessProcessGroup` to create a process group
without considering the global process group in torch.distributed.
It is recommended to create `StatelessProcessGroup`, and then initialize
the data-plane communication (NCCL) between external (train processes)
and vLLM workers.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
pg = StatelessProcessGroup.create(
host=master_address, port=master_port, rank=rank, world_size=world_size
)
with unpinned_nccl_env():
return PyNcclCommunicator(pg, device=device)
def uid_init_process_group(
nccl_unique_id_bytes: bytes,
rank: int,
world_size: int,
device,
) -> "PyNcclCommunicator":
"""Join the NCCL group from pre-shared ``ncclUniqueId`` bytes.
The torch-free rendezvous alternative to `stateless_init_process_group`: no
TCPStore, and therefore no barrier -- every rank must enter concurrently.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
with unpinned_nccl_env():
return PyNcclCommunicator.from_unique_id_bytes(
nccl_unique_id_bytes,
rank=rank,
world_size=world_size,
device=device,
)
def _worker_group_rank(rank_offset: int, parallel_config: "ParallelConfig") -> int:
"""Return this worker's rank in a trainer-to-worker process group."""
dp_rank = parallel_config.data_parallel_index
world_size_per_dp = parallel_config.world_size # TP * PP
rank_within_dp = parallel_config.rank
return rank_offset + dp_rank * world_size_per_dp + rank_within_dp
def worker_init_metadata_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
) -> "StatelessProcessGroup":
"""Join the trainer-to-worker TCP metadata group without constructing NCCL."""
assert init_info.master_address is not None
assert init_info.master_port is not None
return stateless_init_metadata_group(
init_info.master_address,
init_info.master_port,
_worker_group_rank(init_info.rank_offset, parallel_config),
init_info.world_size,
)
def worker_init_process_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
) -> "PyNcclCommunicator":
"""Create the trainer<->worker NCCL group on an inference worker.
Computes a unique rank for this worker across all data-parallel groups and
joins the trainer via whichever rendezvous mode `init_info` carries.
"""
rank = _worker_group_rank(init_info.rank_offset, parallel_config)
device = torch.accelerator.current_device_index()
unique_id_bytes = init_info.nccl_unique_id_bytes
if unique_id_bytes is not None:
return uid_init_process_group(
unique_id_bytes,
rank,
init_info.world_size,
device,
)
# __post_init__ guarantees the TCP pair when there is no unique id.
assert init_info.master_address is not None
assert init_info.master_port is not None
return stateless_init_process_group(
init_info.master_address,
init_info.master_port,
rank,
init_info.world_size,
device=device,
)
def trainer_init(
init_info: NCCLRendezvous | dict,
rank: int = 0,
) -> "PyNcclCommunicator":
"""Initialize NCCL process group for trainer-side weight transfer.
Uses the current CUDA device (torch.accelerator.current_device_index()).
Args:
init_info: Any object carrying the `NCCLRendezvous` fields (a trainer or
worker NCCL init info), or a dict with keys:
- master_address: str
- master_port: int
- world_size: int
rank: This trainer process's rank in the group. The broadcast backends
have a single trainer at rank 0; multi-rank trainers (m2n) occupy
ranks `[0, num_trainer_ranks)` and the workers start after them.
Returns:
PyNcclCommunicator for weight transfer.
"""
if isinstance(init_info, dict):
master_address = init_info["master_address"]
master_port = init_info["master_port"]
world_size = init_info["world_size"]
else:
master_address = init_info.master_address
master_port = init_info.master_port
world_size = init_info.world_size
device = torch.accelerator.current_device_index()
return stateless_init_process_group(
master_address,
master_port,
rank,
world_size,
device,
)
@@ -1,461 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NCCL-based (dense) weight transfer engine."""
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, ClassVar, Self
import torch
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
ParamMeta,
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
WeightTransferEngine,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
worker_init_payload,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
trainer_init as open_trainer_endpoint,
)
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
packed_nccl_broadcast_consumer,
packed_nccl_broadcast_producer,
)
# NCCLWeightTransferInitInfo is re-exported here for convenience; its canonical
# home is nccl_common, shared with the sparse backend.
__all__ = [
"NCCLWeightTransferInitInfo",
"NCCLTrainerInitInfo",
"NCCLWeightTransferUpdateInfo",
"NCCLWeightTransferEngine",
"NCCLTrainerWeightTransferEngine",
]
@dataclass
class NCCLTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for the dense NCCL weight transfer backend.
The sender opens its endpoint as NCCL rank 0, so it needs no `rank_offset`.
`world_size` is the full trainer+worker NCCL group size. `rank` (from
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
The trainer joins over a TCPStore rendezvous (`master_address` +
`master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore
mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a
`nccl_unique_id_b64` payload straight to the inference workers'
`init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than
going through this engine.
`packed` / buffer sizes are the transfer's wire params. The trainer
propagates them to the worker at `trainer_init` so the two sides cannot
disagree. Note this defaults to packed, unlike the worker-side
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
ships a value. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "nccl"
master_address: str
master_port: int
world_size: int
packed: bool = True
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
@dataclass
class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Per-round update info for the dense NCCL weight transfer backend.
Whether the transfer is packed (and the buffer geometry) is a must-agree
wire param carried on the init info (`NCCLTrainerInitInfo` /
`NCCLWeightTransferInitInfo`), not here; this carries only the per-round
parameter metadata.
"""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
def __post_init__(self):
"""Validate that all lists have the same length."""
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
f"`dtype_names` should be of the same size as `names`: "
f"got {len(self.dtype_names)} and {len(self.names)}"
)
if len(self.shapes) != num_params:
raise ValueError(
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
class NCCLWeightTransferEngine(
WeightTransferEngine[NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo]
):
"""Weight transfer engine using NCCL for communication between trainer and workers.
This implementation uses NCCL broadcast operations to transfer dense
checkpoint-format weights from the trainer (rank 0) to all inference workers
in a process group. Received weights are loaded via the model's
`load_weights` using the layerwise reload lifecycle.
"""
# Define backend-specific dataclass types
init_info_cls = NCCLWeightTransferInitInfo
update_info_cls = NCCLWeightTransferUpdateInfo
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
# Set from the trainer-supplied init info at the handshake; defaults are
# only for the (unreachable) receive-before-init case.
self.packed = False
self.packed_buffer_size_bytes = DEFAULT_PACKED_BUFFER_SIZE_BYTES
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
"""Initialize NCCL process group with the trainer and record the
trainer-supplied wire params so the worker decodes exactly as the
trainer encodes.
Args:
init_info: NCCL initialization info containing master address, port,
rank offset, world size, and the packed wire params
"""
self.packed = init_info.packed
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
self.packed_num_buffers = init_info.packed_num_buffers
self.model_update_group = worker_init_process_group(
init_info, self.parallel_config
)
def start_weight_update(self) -> None:
"""Initialize layerwise reloading for the incoming checkpoint weights."""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading after all weights have been received."""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
"""Receive weights from trainer via NCCL broadcast.
Whether to use packed broadcasting (and the buffer geometry) is read
from `self.packed` / `self.packed_*`, set at the init handshake from the
trainer's init info, so it is guaranteed to match how the trainer
encoded.
Args:
update_info: NCCL update info containing parameter names, dtypes,
and shapes
"""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
from vllm.model_executor.model_loader.mtp_validation import (
disable_mtp_completeness_check,
)
with disable_mtp_completeness_check():
if self.packed:
# Build iterator of (name, (shape, dtype)) from update_info
def state_dict_info_iterator():
for name, dtype_name, shape in zip(
update_info.names, update_info.dtype_names, update_info.shapes
):
dtype = getattr(torch, dtype_name)
yield (name, (shape, dtype))
packed_nccl_broadcast_consumer(
iterator=state_dict_info_iterator(),
group=self.model_update_group,
src=0,
post_unpack_func=self.model.load_weights,
buffer_size_bytes=self.packed_buffer_size_bytes,
num_buffers=self.packed_num_buffers,
device=self.device,
)
else:
# Use simple one-by-one broadcasting
for name, dtype_name, shape in zip(
update_info.names, update_info.dtype_names, update_info.shapes
):
dtype = getattr(torch, dtype_name)
weight = torch.empty(shape, dtype=dtype, device=self.device)
self.model_update_group.broadcast(
weight, src=0, stream=torch.cuda.current_stream()
)
self.model.load_weights([(name, weight)])
del weight
def shutdown(self) -> None:
if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None
class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerInitInfo]):
"""Trainer-side NCCL weight transfer engine.
On the sender (rank 0) holds the NCCL communicator and drives the full
update round trip: it runs the inference-side `update_weights` concurrently
with the trainer-side broadcast (both rendezvous inside the same NCCL
calls), then finishes the update. Non-sender trainer ranks hold no
communicator; they only iterate the source to stay in the trainer-side
collective (e.g. FSDP `full_tensor()`) and skip the client RPCs and the
broadcast (all guarded on `is_sender`).
`packed` / buffer sizes come from `NCCLTrainerInitInfo`; the sender
propagates them to the worker at `trainer_init` (on the worker-side init
info), so per-round payloads carry only parameter metadata.
"""
init_info_cls = NCCLTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: WeightSource,
is_sender: bool = True,
packed: bool = True,
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
) -> None:
super().__init__(client=client, source=source, is_sender=is_sender)
self.packed = packed
self.packed_buffer_size_bytes = packed_buffer_size_bytes
self.packed_num_buffers = packed_num_buffers
self.model_update_group: PyNcclCommunicator | None = None
@classmethod
def trainer_init(
cls,
init_info: NCCLTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
if source is None:
raise ValueError("NCCL trainer weight transfer requires a WeightSource.")
engine = cls(
client=client,
source=source,
is_sender=init_info.is_sender,
packed=init_info.packed,
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
packed_num_buffers=init_info.packed_num_buffers,
)
if not engine.is_sender:
# Non-sender trainer ranks aren't part of the transfer NCCL group and
# don't drive the inference side; they only participate in the
# trainer-side gather during send_weights.
return engine
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
worker_init_info = NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
rank_offset=1,
world_size=init_info.world_size,
packed=init_info.packed,
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
packed_num_buffers=init_info.packed_num_buffers,
)
# The inference workers block inside init_weight_transfer_engine waiting
# for the NCCL rendezvous, so we kick that off on a side thread while we
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine,
worker_init_payload(worker_init_info),
)
engine.model_update_group = open_trainer_endpoint(init_info)
future.result() # surface any inference-side init error
return engine
def send_weights(self) -> None:
assert self.source is not None # guaranteed by trainer_init / __init__
source = self.source
# Metadata is declared without gathering. For Megatron it is itself a
# collective, so every rank runs it; only the sender ships it.
meta = source.metadata()
if not self.is_sender:
# Non-sender ranks only join the trainer-side gather collective.
self._broadcast(source, meta)
self._post_send_sync()
return
update_info = NCCLWeightTransferUpdateInfo(
names=[m.name for m in meta],
dtype_names=[str(m.dtype).split(".")[-1] for m in meta],
shapes=[list(m.shape) for m in meta],
)
self.client.start_weight_update()
# update_weights (workers receive) must run concurrently with the
# trainer-side broadcast — both rendezvous inside the same NCCL calls.
exe = ThreadPoolExecutor(max_workers=1)
try:
future = exe.submit(self.client.update_weights, asdict(update_info))
# Cheap best-effort: if update_weights already failed (e.g. a bad
# request rejected before any NCCL call), surface it now instead of
# hanging in broadcast waiting for a peer that will never arrive.
if future.done():
future.result()
self._broadcast(source, meta)
future.result() # surface inference-side errors
finally:
# Never wait for the RPC thread here. If the broadcast raised, the
# worker is still blocked in the matching NCCL call and will never
# return, so joining would turn the error into a permanent hang.
# Let the exception out instead; the transfer group is unusable
# either way and the caller has to tear it down. (The orphaned
# thread is still joined at interpreter exit, so a caller that wants
# to exit cleanly after such a failure must not wait on it.)
exe.shutdown(wait=False)
self.client.finish_weight_update()
self._post_send_sync()
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
"""Iterate the source (materializing each tensor — a collective on all
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
Non-sender ranks only replay the iteration to stay in the collective."""
if not self.is_sender:
for _ in source:
pass
return
assert self.model_update_group is not None, (
"trainer_init() must be called before _broadcast()."
)
pairs = self._checked_iter(source, meta)
if self.packed:
packed_nccl_broadcast_producer(
iterator=pairs,
group=self.model_update_group,
src=0,
post_iter_func=lambda item: item[1],
buffer_size_bytes=self.packed_buffer_size_bytes,
num_buffers=self.packed_num_buffers,
)
else:
stream = torch.cuda.current_stream()
for _name, tensor in pairs:
# NCCL sends `numel` elements straight from `data_ptr()`, so a
# non-contiguous view would ship whatever follows its base
# pointer. Keep the copy referenced until the broadcast is
# enqueued. (The packed path linearizes in `pack_tensors`.)
send = tensor if tensor.is_contiguous() else tensor.contiguous()
self.model_update_group.broadcast(send, src=0, stream=stream)
@staticmethod
def _checked_iter(
source: WeightSource, meta: list[ParamMeta]
) -> Iterator[tuple[str, torch.Tensor]]:
"""Yield the source's pairs, checking each against what the worker was
told to expect.
The worker sizes its receive buffers — and in packed mode cuts its chunk
boundaries — from the update info, which is built from `metadata()`. If
iteration disagrees with it, the two sides split the stream differently
and the transfer hangs in NCCL or loads garbage. Checking here costs one
comparison per parameter and turns that into an error naming the first
divergent parameter. Sender-only: under pipeline parallelism a
non-sender's yielded tensor is not meaningful.
"""
sent = 0
for name, tensor in source:
if sent >= len(meta):
raise ValueError(
f"WeightSource yielded more parameters than metadata() "
f"declared ({len(meta)}); first extra is {name!r}."
)
expected = meta[sent]
if (
name != expected.name
or tensor.dtype != expected.dtype
or tuple(tensor.shape) != expected.shape
):
raise ValueError(
"WeightSource metadata() disagrees with iteration at index "
f"{sent}: declared {expected.name!r} "
f"{expected.dtype} {tuple(expected.shape)}, got {name!r} "
f"{tensor.dtype} {tuple(tensor.shape)}. Both channels must "
"enumerate the same parameters in the same order."
)
sent += 1
yield name, tensor
if sent != len(meta):
raise ValueError(
f"WeightSource yielded {sent} parameters but metadata() "
f"declared {len(meta)}; the worker is waiting for the rest."
)
def _post_send_sync(self) -> None:
"""Wait for this rank's transfer work to land before returning.
Broadcasts are only *enqueued* by `send_weights`: the unpacked path on
the current stream, the packed path on the producer's own streams (which
it drains itself). Waiting here lets a caller mutate parameters, or
start the next step on another stream, as soon as `send_weights`
returns, instead of silently depending on same-stream ordering. Every
rank waits: a non-sender's `full_tensor()` gathers feed the sender's
broadcast, so they must have landed before it may touch its shards.
Unlike IPC there is no cross-rank barrier here. Nothing in this backend
outlives the collective it travelled in (IPC's barrier keeps shared
buffers alive until every consumer has opened them), so a barrier would
only add a dependency on the default process group that this backend
otherwise does not have.
"""
if torch.cuda.is_available():
torch.cuda.current_stream().synchronize()
def shutdown(self) -> None:
if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None
@@ -1,494 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Packed tensor utilities for efficient weight transfer."""
import math
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from functools import cache
from typing import Any
import torch
from torch.multiprocessing.reductions import reduce_tensor
# Default values for packed tensor transfer.
DEFAULT_PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024 # 1GB
DEFAULT_PACKED_NUM_BUFFERS = 2
# Streams are cached across calls: the caching allocator keys cached blocks by the
# stream they were allocated on, so fresh streams at every call would strand the
# packed buffers' reserved memory per call. See gh-52950.
@cache
def _get_streams(device_idx: int, num_buffers: int) -> tuple[torch.cuda.Stream, ...]:
return tuple(torch.cuda.Stream(device=device_idx) for _ in range(num_buffers))
def unpack_tensor(
packed_tensor: torch.Tensor,
names: list[str],
shapes: list[list[int]],
dtypes: list[torch.dtype],
tensor_sizes: list[int],
) -> list[tuple[str, torch.Tensor]]:
"""Unpack a packed uint8 tensor into a list of named tensors.
The returned tensors are **views** of ``packed_tensor`` (the
``.contiguous()`` call is a no-op on already-contiguous row-slices).
If ``packed_tensor`` lives in storage that may be reused — e.g. a
reused CUDA IPC buffer — callers must clone the results before the
underlying storage is overwritten.
Args:
packed_tensor: The packed torch.uint8 tensor to unpack
names: List of tensor names
shapes: List of tensor shapes
dtypes: List of tensor dtypes
tensor_sizes: List of tensor sizes in bytes
"""
unpacked_tensors = packed_tensor.split(tensor_sizes)
return [
(name, tensor.contiguous().view(dtype).view(shape))
for name, shape, dtype, tensor in zip(names, shapes, dtypes, unpacked_tensors)
]
@dataclass
class PackedChunk:
"""Result of packing tensors into a single contiguous uint8 buffer."""
packed_tensor: torch.Tensor
names: list[str]
shapes: list[list[int]]
dtypes: list[torch.dtype]
tensor_sizes: list[int]
def pack_tensors(
iterator: Iterator[tuple[str, torch.Tensor]],
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
buffer_size_bytes: int,
tensor_list: list[torch.Tensor] | None = None,
current_size: int = 0,
) -> PackedChunk | None:
"""Pack tensors from an iterator into a single contiguous uint8 buffer.
Consumes from the iterator until the accumulated size exceeds
buffer_size_bytes or the iterator is exhausted, then returns a
PackedChunk. Returns None if no tensors were consumed.
Args:
iterator: Iterator of (name, tensor) pairs
post_iter_func: Applied to each item before linearizing to uint8
buffer_size_bytes: Max bytes before flushing
tensor_list: Pre-existing tensor list to append to (for NCCL
multi-buffer reuse). If None, a fresh list is created.
current_size: Byte count already accumulated in tensor_list
"""
if tensor_list is None:
tensor_list = []
names: list[str] = []
shapes: list[list[int]] = []
dtypes: list[torch.dtype] = []
tensor_sizes: list[int] = []
total_bytes = current_size
while True:
try:
item = next(iterator)
except StopIteration:
break
name, orig_tensor = item
# Apply post processing and convert to linearized uint8 tensor
tensor = post_iter_func(item).contiguous().view(-1).view(torch.uint8)
if tensor.numel() > buffer_size_bytes:
import warnings
warnings.warn(
f"Tensor '{name}' has size {tensor.numel()} bytes, which "
f"exceeds buffer_size_bytes={buffer_size_bytes}.",
stacklevel=2,
)
tensor_list.append(tensor)
names.append(name)
shapes.append(list(orig_tensor.shape))
dtypes.append(orig_tensor.dtype)
tensor_sizes.append(tensor.numel())
total_bytes += tensor.numel()
if total_bytes > buffer_size_bytes:
break
if not tensor_list:
return None
packed = torch.cat(tensor_list, dim=0)
del tensor_list
return PackedChunk(
packed_tensor=packed,
names=names,
shapes=shapes,
dtypes=dtypes,
tensor_sizes=tensor_sizes,
)
# ── NCCL packed broadcast ──────────────────────────────────────────────
def packed_nccl_broadcast_producer(
iterator: Iterator[tuple[str, torch.Tensor]],
group: Any,
src: int,
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
) -> None:
"""Broadcast tensors in a packed manner from trainer to workers.
Args:
iterator: Iterator of model parameters. Returns a tuple of (name, tensor)
group: Process group (PyNcclCommunicator)
src: Source rank (0 in current implementation)
post_iter_func: Function to apply to each (name, tensor) pair before
packing, should return a tensor
buffer_size_bytes: Size in bytes for each packed tensor buffer.
Both producer and consumer must use the same value.
num_buffers: Number of buffers for double/triple buffering.
Both producer and consumer must use the same value.
"""
streams = _get_streams(torch.accelerator.current_device_index(), num_buffers)
# Keep references to in-flight chunks so their packed_tensors
# aren't freed while an async broadcast is still reading them.
in_flight: list[PackedChunk | None] = [None] * num_buffers
buffer_idx = 0
while True:
# Synchronize the current stream
streams[buffer_idx].synchronize()
# Previous chunk on this buffer slot is now safe to free
in_flight[buffer_idx] = None
# Start tasks for the new buffer in a new stream
with torch.cuda.stream(streams[buffer_idx]):
chunk = pack_tensors(iterator, post_iter_func, buffer_size_bytes)
if chunk is None:
break
# Pack the tensors and call broadcast collective
group.broadcast(chunk.packed_tensor, src=src)
# Hold reference until this stream is synchronized
in_flight[buffer_idx] = chunk
# Move to the next buffer
buffer_idx = (buffer_idx + 1) % num_buffers
# Drain every slot before returning.
for stream in streams:
stream.synchronize()
def packed_nccl_broadcast_consumer(
iterator: Iterator[tuple[str, tuple[list[int], torch.dtype]]],
group: Any,
src: int,
post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None],
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
device: torch.device | str = "cuda",
) -> None:
"""Consume packed tensors and unpack them into a list of tensors.
Args:
iterator: Iterator of parameter metadata. Returns (name, (shape, dtype))
group: Process group (PyNcclCommunicator)
src: Source rank (0 in current implementation)
post_unpack_func: Function to apply to each list of (name, tensor) after
unpacking
buffer_size_bytes: Size in bytes for each packed tensor buffer.
Both producer and consumer must use the same value.
num_buffers: Number of buffers for double/triple buffering.
Both producer and consumer must use the same value.
device: Device for the receive buffers. Must match the device the NCCL
communicator was created on (the worker's assigned device).
"""
target_packed_tensor_size = buffer_size_bytes
streams = _get_streams(torch.accelerator.current_device_index(), num_buffers)
buffer_idx = 0
packing_tensor_meta_data: list[list[tuple[str, list[int], torch.dtype, int]]] = [
[] for _ in range(num_buffers)
]
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
packed_tensors: list[torch.Tensor] = [
torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers)
]
while True:
# Synchronize the current stream
streams[buffer_idx].synchronize()
with torch.cuda.stream(streams[buffer_idx]):
# Initialize the packing tensor meta data
packing_tensor_meta_data[buffer_idx] = []
packing_tensor_sizes[buffer_idx] = 0
try:
# Form a packed tensor
while True:
name, (shape, dtype) = next(iterator)
tensor_size = math.prod(shape) * dtype.itemsize
packing_tensor_meta_data[buffer_idx].append(
(name, shape, dtype, tensor_size)
)
packing_tensor_sizes[buffer_idx] += tensor_size
if packing_tensor_sizes[buffer_idx] > target_packed_tensor_size:
break
# Create a packed tensor and broadcast it
packed_tensors[buffer_idx] = torch.empty(
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Load the packed tensor into the model
names, shapes, dtypes, tensor_sizes = zip(
*packing_tensor_meta_data[buffer_idx]
)
post_unpack_func(
unpack_tensor(
packed_tensors[buffer_idx],
list(names),
list(shapes),
list(dtypes),
list(tensor_sizes),
)
)
# Move to the next buffer
buffer_idx = (buffer_idx + 1) % num_buffers
except StopIteration:
# Do the last broadcast if there are remaining tensors
if len(packing_tensor_meta_data[buffer_idx]) > 0:
# Create a packed tensor and broadcast it
packed_tensors[buffer_idx] = torch.empty(
packing_tensor_sizes[buffer_idx],
dtype=torch.uint8,
device=device,
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Load the packed tensor into the model
names, shapes, dtypes, tensor_sizes = zip(
*packing_tensor_meta_data[buffer_idx]
)
post_unpack_func(
unpack_tensor(
packed_tensors[buffer_idx],
list(names),
list(shapes),
list(dtypes),
list(tensor_sizes),
)
)
break
# Drain every slot before returning.
for stream in streams:
stream.synchronize()
# ── IPC packed transfer ────────────────────────────────────────────────
@dataclass
class PackedIpcChunk:
"""Metadata and IPC handle for a single packed chunk."""
names: list[str]
shapes: list[list[int]]
dtype_names: list[str]
tensor_sizes: list[int]
ipc_handle: dict[str, tuple]
def packed_ipc_producer(
iterator: Iterator[tuple[str, torch.Tensor]],
gpu_uuid: str,
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
) -> Iterator[PackedIpcChunk]:
"""Pack tensors into a reusable IPC buffer and yield handles.
Allocates a single GPU buffer of ``buffer_size_bytes`` and registers
it for IPC once via ``reduce_tensor``. Each chunk's packed data is
copied into this buffer before yielding, so only one IPC-shared
allocation exists for the lifetime of the transfer.
Callers **must** ensure the consumer has finished reading the buffer
(e.g. ``ray.get`` returned) before resuming the generator for the
next chunk.
Args:
iterator: Iterator of (name, tensor) pairs.
gpu_uuid: Physical GPU UUID string for this rank.
post_iter_func: Applied to each (name, tensor) before packing.
buffer_size_bytes: Exact capacity of the reusable IPC buffer.
Every chunk is guaranteed to fit within this size. A
``ValueError`` is raised if any single tensor exceeds it.
"""
ipc_buffer = torch.empty(buffer_size_bytes, dtype=torch.uint8, device="cuda")
_, ipc_args = reduce_tensor(ipc_buffer)
names: list[str] = []
shapes: list[list[int]] = []
dtypes: list[torch.dtype] = []
tensor_sizes: list[int] = []
total_bytes = 0
for name, orig_tensor in iterator:
flat = (
post_iter_func((name, orig_tensor)).contiguous().view(-1).view(torch.uint8)
)
if flat.numel() > buffer_size_bytes:
raise ValueError(
f"Tensor '{name}' has size {flat.numel()} bytes, "
f"which exceeds buffer_size_bytes={buffer_size_bytes}. "
f"Increase buffer_size_bytes to at least {flat.numel()}."
)
if total_bytes and total_bytes + flat.numel() > buffer_size_bytes:
# Drain queued copies so the consumer sees a fully-written buffer.
torch.cuda.current_stream().synchronize()
yield PackedIpcChunk(
names=names,
shapes=shapes,
dtype_names=[str(d).split(".")[-1] for d in dtypes],
tensor_sizes=tensor_sizes,
ipc_handle={gpu_uuid: ipc_args},
)
# Rebind to fresh lists so the yielded chunk's metadata is
# not mutated while the consumer is still reading.
names, shapes, dtypes, tensor_sizes = [], [], [], []
total_bytes = 0
ipc_buffer[total_bytes : total_bytes + flat.numel()].copy_(flat)
names.append(name)
shapes.append(list(orig_tensor.shape))
dtypes.append(orig_tensor.dtype)
tensor_sizes.append(flat.numel())
total_bytes += flat.numel()
if total_bytes:
torch.cuda.current_stream().synchronize()
yield PackedIpcChunk(
names=names,
shapes=shapes,
dtype_names=[str(d).split(".")[-1] for d in dtypes],
tensor_sizes=tensor_sizes,
ipc_handle={gpu_uuid: ipc_args},
)
class PackedBufferImporter:
"""One-slot cache for the consumer-side mapping of a packed IPC buffer.
torch's cross-process refcount pairs one ``reduce_tensor`` export with
one consumer rebuild-release cycle. The producer exports its buffer once
and ships the same args with every chunk, so rebuilding and releasing
per chunk decrements the counter once per chunk, drives it negative,
and the dropped buffer is then never reclaimable by ``ipc_collect`` on
the producer side (it leaks one buffer per transfer). Caching the
rebuilt buffer keyed by the rebuild args restores the pairing: chunks
of one transfer reuse the mapping (it aliases producer memory, so it
always reads the current chunk's bytes), and replacing the entry when
the next export arrives — or ``close()`` — releases the previous
mapping exactly once.
The importer that consumes a multi-chunk transfer must be one object
reused across chunks; a fresh importer per chunk reintroduces the
per-chunk release. ``IPCWeightTransferEngine`` owns one instance per
engine and closes it on shutdown.
"""
def __init__(self) -> None:
self._entry: tuple[tuple, torch.Tensor] | None = None
def rebuild(self, list_args: list) -> torch.Tensor:
from torch.multiprocessing.reductions import rebuild_cuda_tensor
key = tuple(list_args)
if self._entry is not None and self._entry[0] == key:
return self._entry[1]
packed = rebuild_cuda_tensor(*list_args)
self._entry = (key, packed)
return packed
def close(self) -> None:
"""Release the held mapping (one producer-side refcount decrement)."""
self._entry = None
def packed_ipc_consumer(
ipc_handle: dict[str, tuple],
names: list[str],
shapes: list[list[int]],
dtype_names: list[str],
tensor_sizes: list[int],
device_index: int,
importer: "PackedBufferImporter | None" = None,
) -> list[tuple[str, torch.Tensor]]:
"""Unpack a single packed IPC chunk into named tensors.
Reconstructs the packed buffer via rebuild_cuda_tensor, unpacks
into individual tensors, and clones each into independent storage
before returning.
The clone is intentional: the producer reuses one IPC buffer across
chunks, so any tensor view that aliases the buffer would observe the
*next* chunk's bytes as soon as the producer's generator is resumed.
Callers that retain references past their own update_weights call
(notably vLLM's layerwise reload, which buffers ``bound_args`` for
replay in ``_layerwise_process``) would otherwise replay against
stale data and silently corrupt multi-chunk weight transfers.
Args:
ipc_handle: Mapping of GPU UUID to rebuild_cuda_tensor args tuple
names: Parameter names in the packed buffer
shapes: Parameter shapes
dtype_names: Parameter dtype name strings (e.g. "float16")
tensor_sizes: Size in bytes of each parameter in the packed buffer
device_index: Local CUDA device index
importer: Import cache that must be shared across all chunks of one
export so the buffer mapping is opened and released exactly once
(see ``PackedBufferImporter``). ``None`` builds an ephemeral one,
which is only safe for single-chunk consumption.
"""
props = torch.cuda.get_device_properties(device_index)
physical_gpu_id = str(props.uuid)
if physical_gpu_id not in ipc_handle:
raise ValueError(
f"IPC handle not found for GPU UUID {physical_gpu_id}. "
f"Available UUIDs: {list(ipc_handle.keys())}"
)
args = ipc_handle[physical_gpu_id]
list_args = list(args)
list_args[6] = device_index
if importer is None:
importer = PackedBufferImporter()
packed = importer.rebuild(list_args)
content_size = sum(tensor_sizes)
packed = packed[:content_size]
dtypes = [getattr(torch, dn) for dn in dtype_names]
return [
(name, t.clone())
for name, t in unpack_tensor(packed, names, shapes, dtypes, tensor_sizes)
]
@@ -1,352 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared pieces of the sharded-RDT backend: the op-chain allowlist, buffer
sizing and the minimum Ray version, plus ``RdtRouter`` — consumer-only, since
routing is a consumer decision.
The gather-group partition itself is ``base.layerwise_groups``: it defines what a
group index means for any ``WeightSource``, not just this transport.
"""
from collections.abc import Callable
from typing import Any
import torch
# The op-chain contract, with two enforcers. The consumer's ``FakeRDTTensor``
# intercepts exactly these methods during the bake and records the string name;
# anything else reaches ``__torch_dispatch__`` and raises, so a loader needing
# real data fails at init rather than transferring wrong bytes. The producer
# replays a chain as ``getattr(tensor, op)(...)`` and refuses anything outside
# ``ALLOWED_OPS``, so a spoofed consumer cannot invoke arbitrary methods.
#
# Every entry must be a pure view; ``to`` in particular must never be added, as
# it is the dtype/device escape the bake exists to reject. Both sets derive from
# this one table so they cannot disagree.
SUPPORTED_OPS: dict[Callable, str] = {
torch.Tensor.narrow: "narrow",
torch.Tensor.view: "view",
torch.Tensor.reshape: "reshape",
torch.Tensor.__getitem__: "__getitem__",
torch.Tensor.unsqueeze: "unsqueeze",
torch.Tensor.squeeze: "squeeze",
torch.Tensor.transpose: "transpose",
torch.Tensor.t: "t",
torch.Tensor.permute: "permute",
torch.Tensor.flatten: "flatten",
torch.Tensor.contiguous: "contiguous",
torch.Tensor.chunk: "chunk",
# Multi-return like chunk: one child per output, trailing __getitem__(i).
torch.Tensor.unbind: "unbind",
}
ALLOWED_OPS = frozenset(SUPPORTED_OPS.values())
# Ray Direct Transport landed in halves: the actor opt-in
# ``enable_tensor_transport`` in 2.49, and ``ray.experimental``'s
# ``register_nixl_memory`` / ``set_target_for_ref`` — which this transport calls
# directly — in 2.55. The floor is the tested version rather than the earliest
# one that could import; 2.55 has never been run against.
RDT_MIN_RAY_VERSION = "2.56.0"
def check_ray_rdt_version() -> None:
"""Refuse an installed Ray older than the one this backend is tested on.
vLLM does not depend on Ray, so there is no pin to carry this. Without the
check the failure is an opaque option-validation error out of
``.options(enable_tensor_transport=True)`` (below 2.49) or an ImportError
raised deep in the first pull, long after init reported success (below 2.55).
Raises:
ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``.
"""
import importlib.metadata
from packaging import version
required = version.parse(RDT_MIN_RAY_VERSION)
current = version.parse(importlib.metadata.version("ray"))
if current < required:
raise ValueError(
f"The 'sharded_rdt' weight transfer backend requires Ray "
f">= {required}, the version it is tested against; Ray Direct "
f"Transport (ray.experimental.register_nixl_memory / "
f"set_target_for_ref) needs at least 2.55. Found {current}. "
f"Run `pip install -U 'ray>={required}'`."
)
def register_nixl_memory(tensor: torch.Tensor) -> None:
"""Register a tensor with Ray using the platform's NIXL package."""
from vllm.distributed.nixl_utils import alias_nixl_for_ray
alias_nixl_for_ray()
from ray.experimental import register_nixl_memory as ray_register_nixl_memory
ray_register_nixl_memory(tensor)
def assign_producer_indices(
num_producers: int, num_consumers: int, consumer_idx: int
) -> list[int]:
"""Producers (global indices) that consumer ``consumer_idx`` binds."""
p = max(1, num_producers)
c = max(1, num_consumers)
if p >= c:
return list(range(consumer_idx * p // c, (consumer_idx + 1) * p // c))
return [consumer_idx * p // c]
class RdtRouter:
"""Decides which producer serves each weight name, and — once bound — sends.
Ownership is per NAME. ``owner_sets`` holds the few distinct producer sets
that occur, and ``name_owner_class[i]`` indexes it for ``names[i]``; a name
determines both its owner set and its gather group, so neither appears in the
routing API. Empty tables mean every producer holds everything.
Any placement is expressible this way: a pipeline stage is a set of names
sharing one owner set, an expert is a name whose owner set is a single rank,
and a group produced by two stages is just two classes inside one group.
Both engines derive the same tables from the same wire data, so they agree on
who serves what. Disagreement is not a wrong answer but a hang or a loud
misroute: a pull sent to a producer that never gathered the name trips its
served-names guard.
Routing is consumer-only. The trainer publishes what it holds and answers
whatever arrives; it never asks who serves what.
"""
def __init__(
self,
num_producers: int,
num_consumers: int,
owner_sets: list[list[int]] | None = None,
name_owner_class: list[int] | None = None,
names: list[str] | None = None,
group_lens: list[int] | None = None,
workers_per_replica: int = 0,
) -> None:
"""Build the routing tables from the wire data both engines receive.
The four table arguments travel together and default together: with all
of them empty the router degrades to "every producer holds everything".
Args:
num_producers: Trainer ranks. Owner indices are positions in this
range, and ``validate`` rejects any that fall outside it.
num_consumers: Inference workers across the whole fleet. Fixes
the id space; the block carve uses ``workers_per_replica``
of it.
owner_sets: The distinct producer sets that occur, one row per
owner class; each row is deduplicated and sorted here. Empty
means a single class owning every producer.
name_owner_class: Parallel to ``names`` — ``name_owner_class[i]``
indexes ``owner_sets`` for ``names[i]``. A name with no entry
falls back to class 0.
names: Every parameter name, in group-major order: concatenating
the gather groups reproduces this list exactly. The other two
tables are keyed by this order.
group_lens: Length of each gather group, consecutive over
``names``, so they sum to ``len(names)``. Fixes name -> group
and the per-group owner union the free barrier fans out to.
workers_per_replica: Consumers per inference DEPLOYMENT. The block
carve spreads this many consumers over an owner set and every
deployment reuses that carve, so the same worker of each
deployment resolves one producer (see ``producer_for``). 0
means one deployment: carve over the whole fleet.
Raises:
ValueError: ``workers_per_replica`` does not divide
``num_consumers``, so the deployments are not uniform and two
workers of one deployment would share a block index.
"""
self.num_producers = max(1, num_producers)
self.num_consumers = max(1, num_consumers)
# Width of the block carve: one deployment's worth of consumers, so
# deployments overlay rather than spread. See ``producer_for``.
self._block_consumers = min(
self.num_consumers, max(1, workers_per_replica or self.num_consumers)
)
if self.num_consumers % self._block_consumers:
raise ValueError(
f"workers_per_replica={workers_per_replica} does not divide "
f"num_consumers={self.num_consumers}; the inference deployments "
f"must be uniform for the per-deployment block carve to be "
f"well defined."
)
self._owner_sets = (
[sorted(set(row)) for row in owner_sets]
if owner_sets
else [list(range(self.num_producers))]
)
names = list(names or [])
classes = list(name_owner_class or [])
self._class_of = {
n: (classes[i] if i < len(classes) else 0) for i, n in enumerate(names)
}
# name -> gather group, and the per-group owner union the free barrier
# fans out to. Both are pure functions of the tables, so precompute once.
self._group_of: dict[str, int] = {}
self._group_owners: list[list[int]] = []
pos = 0
for gi, glen in enumerate(group_lens or []):
union: set[int] = set()
for n in names[pos : pos + glen]:
self._group_of[n] = gi
c = self._class_of.get(n, 0)
# Bounds-checked: a bad class must surface from validate() as a
# named ValueError, not as an IndexError from this precompute.
if 0 <= c < len(self._owner_sets):
union.update(self._owner_sets[c])
pos += glen
self._group_owners.append(sorted(union))
# Bound at init on the consumer; the rule methods never touch these.
self._actors: list[Any] = []
self._produce_methods: list[Any] = []
self.consumer_id = 0
# ---------------- rules (pure; safe to use unbound) ----------------
def class_of(self, name: str) -> int:
"""The name's owner class — the planner's bucketing key, since all names
of a chunk must share one producer."""
return self._class_of.get(name, 0)
def owners(self, name: str) -> list[int]:
"""Every producer holding ``name``."""
return list(self._owner_sets[self.class_of(name)])
def group_owners(self, group_idx: int) -> list[int]:
"""Every producer holding any name of ``group_idx``.
The one group-keyed rule, because the free barrier is per group: a
consumer signals ``free_group(gi)`` at each of these and the producer
counts signals against its live-consumer total. Names route; groups free.
"""
if not self._group_owners:
return list(range(self.num_producers))
return list(self._group_owners[group_idx])
def producer_for(self, consumer_id: int, name: str) -> int:
"""The single producer ``consumer_id`` pulls ``name`` from.
Blocks consumers across the name's owner set with the same rule that
binds producers globally, then rotates by the name's GROUP index so a
consumer spreads its groups over its block instead of hammering one NIC.
Rotating per group (not per name) keeps every name of a chunk on one
producer, which is what lets a chunk be a single pull.
The carve is over ONE DEPLOYMENT (this consumer's index within its own
deployment, over ``workers_per_replica`` of them), so the same worker of
every deployment resolves the same producer for every name. Carving over
the whole fleet instead spreads the multi-owner names of different
deployments onto different producers while single-owner names still
route by ownership, so a producer serves several distinct worker
indices, each needing its own serve ring, and none of them can share a
slot. With one deployment the width is the fleet, so this is the plain
block rule.
"""
own = self.owners(name)
if not own:
raise ValueError(f"{name!r} has no owner")
block = assign_producer_indices(
len(own), self._block_consumers, consumer_id % self._block_consumers
)
return own[block[self._group_of.get(name, 0) % len(block)]]
def validate(self) -> None:
"""Check the ownership tables can be served.
Raises:
ValueError: an owner set is empty or out of range, or a class index
does not resolve.
"""
for c, row in enumerate(self._owner_sets):
if not row:
raise ValueError(f"owner set {c} is empty: no producer holds it")
bad = [p for p in row if not 0 <= p < self.num_producers]
if bad:
raise ValueError(f"owner set {c} out of range: {bad}")
for name, c in self._class_of.items():
if not 0 <= c < len(self._owner_sets):
raise ValueError(
f"{name!r} has owner class {c}, but only "
f"{len(self._owner_sets)} owner set(s) were shipped"
)
# ---------------- send surface (needs the binding) ----------------
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
never appended to: every owner index is a position in these lists, so a
rejoining engine that re-inits must not shift them."""
if len(actors) != len(produce_methods):
raise ValueError("actors and produce_methods must be parallel")
self._actors = list(actors)
self._produce_methods = list(produce_methods)
self.consumer_id = consumer_id
def _bound(self) -> None:
if not self._produce_methods:
raise RuntimeError("RdtRouter has no producers bound; call bind() first.")
def pull(self, owner: int, keys: list, seq: int) -> Any:
"""Issue one packed pull to ``owner``.
The consumer id keys the producer's serve ring; without it every worker
is served out of ring 0 and concurrent pulls overwrite each other's blob.
``seq`` is this call's index in the stream to ``owner`` (see ``_Chunk``):
it selects the serve slot in ISSUE order, so a slot is never repacked
while the read of its previous contents is still in flight.
"""
self._bound()
return self._produce_methods[owner].remote(
keys, consumer_id=self.consumer_id, seq=seq
)
def free_group(self, group_idx: int) -> list:
"""Signal the group done at every producer holding any of its names."""
self._bound()
return [
self._actors[p].free_group.remote(group_idx)
for p in self.group_owners(group_idx)
]
def reserve_serve_buffers(
self, bytes_by_producer: list[int], plan_digests: list[str] | None = None
) -> list:
"""Ask each producer to pre-register a serve ring sized to the most this
consumer will pull from it.
``plan_digests[p]`` describes the chunks this consumer pulls from
producer ``p``, in pull order. A producer that shares one serve ring
across the consumers of several deployments compares it across them, so
a fleet whose deployments are not identical fails at init instead of
stalling mid-sync. A producer that shares nothing ignores it.
"""
self._bound()
digests: list = (
list(plan_digests) if plan_digests else [None] * len(bytes_by_producer)
)
return [
self._actors[p].reserve_serve_buffer.remote(
self.consumer_id, nb, digests[p]
)
for p, nb in enumerate(bytes_by_producer)
if nb > 0
]
def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
"""Size a NIXL buffer / ring slot for ``nbytes``: the max of the request, an
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
and the producer's serve rings."""
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
File diff suppressed because it is too large Load Diff
@@ -1,300 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Op-chain recording for the sharded-RDT backend.
The consumer asks the trainer for the exact slice a worker consumes, described as
an op chain replayed on the trainer's live tensor. ``FakeRDTTensor`` builds the
chain by intercepting the model's own weight loaders; ``copy_`` is its data sink,
``BakeSink`` — the dry run records how each slice would be fetched and where it
lands.
Chains are built against meta tensors, so nothing here touches Ray or a GPU.
"""
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
import torch
from vllm.distributed.weight_transfer.sharded_rdt_common import SUPPORTED_OPS
# ("op_name", positional_args, sorted_kwargs_items). Entries must be hashable so
# the chain can serve as a FetchKey.
OpSpec = tuple[str, tuple[Any, ...], tuple[tuple[str, Any], ...]]
OpChain = tuple[OpSpec, ...]
FetchKey = tuple[str, OpChain]
def _freeze_kwargs(kwargs: dict[str, Any]) -> tuple[tuple[str, Any], ...]:
"""Sort kwargs into a tuple of items for hashable storage in OpSpec."""
return tuple(sorted(kwargs.items()))
class _UnsupportedFakeOp(NotImplementedError):
"""Raised when a weight loader does something to a FakeRDTTensor that cannot
be expressed as a slice request.
Surfaced as NotImplementedError so callers can distinguish "this backend
can't handle this loader" from genuine bugs.
"""
@dataclass
class _Scatter:
"""One recorded scatter: pull ``src``, copy it into ``layer``'s
``param_name`` at the recorded strided region. The bake's output unit; see
the engine module's Data flow.
Self-contained so replay needs no lookups. Destination geometry is read off
the meta view and rebuilt each sync as ``as_strided(shape, stride, offset)``
-- ``layer`` is resolved to a param at replay time, never baked, since every
sync re-materializes fresh tensors.
``shape`` and ``dtype`` also size the slice ON THE WIRE, so ``dtype`` is the
PRODUCED dtype (the fake's after its op chain), not the source name's:
``view(dtype)`` is allowlisted, and taking the source's would size the slice
with the wrong itemsize and carve the packed blob differently on the two
sides.
"""
layer: Any
param_name: str
src: FetchKey
offset: int
shape: tuple[int, ...]
stride: tuple[int, ...]
dtype: torch.dtype
def _meta_copy_(dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor:
"""Fire ``dest.copy_`` from a zero-storage meta source of ``src``'s geometry.
Moves no data but still counts against the layer's loaded numel, which drives
``_layerwise_process``; skipping it would leave the layer looking unloaded
forever.
A non-meta ``dest`` is one the layerwise reload never moved there (the
``SKIP_LOAD_TENSORS`` set, e.g. GLM's router bias). Torch forbids
``real.copy_(meta)``, and skipping the count is consistent -- ``get_layer_size``
excludes the same set. The caller still RECORDS the copy, so the replay writes
the real param.
"""
if not dest.is_meta:
return dest
meta_src = torch.empty(src.shape, dtype=src.dtype, device="meta")
with torch._C.DisableTorchFunctionSubclass():
return dest.copy_(meta_src)
@dataclass
class BakeSink:
"""``copy_`` sink for the dry-run bake: record how each slice would be fetched
and where it would land, and move nothing.
``_install_recording_stamps`` sets ``current = (leaf_module, param_name)``
around each loader so ``accept_copy`` can attribute the copy. An unstamped
``copy_`` cannot be attributed and stays unrecorded, so its module fails the
coverage gate and takes the plain load. ``copies_by_layer`` is keyed by module
object, so iterating it yields each leaf module once.
"""
copies_by_layer: "dict[Any, list[_Scatter | None]]" = field(
default_factory=lambda: defaultdict(list)
)
current: "tuple[Any, str] | None" = None
# Source names whose ``copy_`` fired during the bake. Names not here never
# moved data (e.g. experts owned by another EP rank) and are skipped; names
# here but unbaked fail the plan build.
copied_names: "set[str]" = field(default_factory=set)
def accept_copy(self, dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor:
self.copied_names.add(src._name)
if self.current is not None:
layer, param_name = self.current
if dest.numel() != src.numel():
# The packed layout sizes each slice from the DESTINATION shape,
# while the producer packs what the replayed chain produces. A
# broadcasting copy_ makes those two lengths differ, and neither
# side can notice: the consumer's buffer view is exactly
# prod(dest.shape) elements, so it reshapes cleanly over bytes
# the producer laid out at different offsets. Refuse at bake.
raise _UnsupportedFakeOp(
f"FakeRDTTensor: broadcasting copy_ into {param_name!r} of "
f"{type(layer).__name__} ({src.numel()} elements from "
f"{src._name!r} into {dest.numel()}). The sharded RDT "
"backend sends exactly the slice the loader asks for, so "
"the source and destination must have equal numel."
)
self.copies_by_layer[layer].append(
_Scatter(
layer=layer,
param_name=param_name,
src=src._key(),
offset=dest.storage_offset(),
shape=tuple(dest.shape),
stride=tuple(dest.stride()),
dtype=src.dtype,
)
)
return _meta_copy_(dest, src)
class FakeRDTTensor(torch.Tensor):
"""Zero-storage tensor that records how to fetch a weight slice.
``_make_wrapper_subclass`` gives it shape/dtype/device without storage. Every
op in ``SUPPORTED_OPS`` returns a child with the spec appended; ``copy_``
delegates to the installed sink. Anything else reaches ``__torch_dispatch__``
and raises ``_UnsupportedFakeOp``, so failures are loud rather than silently
fetching the wrong bytes.
"""
# Declared at class scope so mypy can infer attribute types on
# instances built via ``_make_wrapper_subclass`` (where ``__new__``
# returns a tensor it can't annotate as ``self``).
_name: str
_ops: OpChain
# Handles this fake's copy_. ``None`` only on bare construction.
_sink: "BakeSink | None"
@staticmethod
def __new__(
cls,
name: str,
shape: torch.Size,
dtype: torch.dtype,
device: torch.device,
ops: OpChain = (),
sink: "BakeSink | None" = None,
) -> "FakeRDTTensor":
t = torch.Tensor._make_wrapper_subclass(
cls,
shape,
dtype=dtype,
device=device,
requires_grad=False,
)
t._name = name
t._ops = tuple(ops)
t._sink = sink
return t
def _key(self) -> FetchKey:
return (self._name, self._ops)
def _make_child(
self,
new_shape: torch.Size,
new_dtype: torch.dtype,
*new_ops: OpSpec,
) -> "FakeRDTTensor":
"""Append one or more ops to the chain and return a fresh child.
Variadic so multi-return ops (e.g. chunk) can append both the base
op and an indexing op in a single call.
"""
return FakeRDTTensor(
name=self._name,
shape=new_shape,
dtype=new_dtype,
device=self.device,
ops=self._ops + new_ops,
sink=self._sink,
)
def _meta(self) -> torch.Tensor:
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
itself computes post-op geometry rather than us reimplementing it."""
return torch.empty(self.shape, dtype=self.dtype, device="meta")
@classmethod
def __torch_function__(
cls,
func,
types,
args=(),
kwargs=None,
):
kwargs = kwargs or {}
# Checked before the allowlist: copy_ must not be recorded into the chain.
if func is torch.Tensor.copy_:
dest = args[0]
src = args[1] if len(args) > 1 else kwargs.get("src")
if isinstance(src, cls) and src._sink is not None:
return src._sink.accept_copy(dest, src)
# Allowlisted slice/view/shape ops: append to chain and return child.
op_name = SUPPORTED_OPS.get(func)
if op_name is not None:
self_ = args[0]
if isinstance(self_, cls):
rest = tuple(args[1:])
return cls._intercept(self_, func, op_name, rest, kwargs)
# Metadata reads are answered by the wrapper subclass and stop here; ops
# that need data fall through to __torch_dispatch__, which raises.
with torch._C.DisableTorchFunctionSubclass():
return func(*args, **kwargs)
@classmethod
def _intercept(
cls,
self_: "FakeRDTTensor",
func: Callable,
op_name: str,
args: tuple,
kwargs: dict,
):
"""Append the op and return a child, or a tuple of children for
multi-return ops. Each child's geometry comes from running the op on a
meta tensor."""
meta = self_._meta()
with torch._C.DisableTorchFunctionSubclass():
meta_result = func(meta, *args, **kwargs)
base_op: OpSpec = (op_name, tuple(args), _freeze_kwargs(kwargs))
# Single-tensor result: one child carrying base_op alone.
if isinstance(meta_result, torch.Tensor):
return self_._make_child(meta_result.shape, meta_result.dtype, base_op)
# Multi-return: one child per output, each carrying base_op plus a
# trailing __getitem__(i) so the replay can index the result.
if isinstance(meta_result, tuple | list):
return tuple(
self_._make_child(
m.shape,
m.dtype,
base_op,
("__getitem__", (i,), ()),
)
for i, m in enumerate(meta_result)
)
# Non-tensor result (e.g. .item() snuck into the allowlist).
raise _UnsupportedFakeOp(
f"FakeRDTTensor: {op_name!r} returned a non-tensor "
f"({type(meta_result).__name__}); cannot defer."
)
@classmethod
def __torch_dispatch__(cls, func, types, args=(), kwargs=None):
kwargs = kwargs or {}
# A fake at the aten level means an unsupported loader op. Name the op and
# the chain; materializing instead would mask a correctness bug.
for arg in (*args, *kwargs.values()):
if isinstance(arg, cls):
raise _UnsupportedFakeOp(
f"FakeRDTTensor: unsupported op {func} reached "
f"__torch_dispatch__ on fake {arg._name!r} "
f"(chain={arg._ops}). Supported ops are: "
f"{sorted(SUPPORTED_OPS.values())}, plus copy_. "
"Loaders that need .to(), .float(), .item(), arithmetic, "
"bool-mask indexing, or .data access are not supported by "
"the sharded RDT backend."
)
return func(*args, **kwargs)
File diff suppressed because it is too large Load Diff
@@ -1,377 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse NCCL weight transfer engine.
Sparse patches use checkpoint names, shapes, and flat indices. The model's native
weight loader maps them to rank-local runtime parameters, including TP shards and
packed parameters.
"""
from collections.abc import Iterable
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, ClassVar, Self
import torch
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
WeightTransferEngine,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
worker_init_payload,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
trainer_init as open_trainer_endpoint,
)
from vllm.model_executor.model_loader.checkpoint_weight_patch import (
CheckpointWeightPatch,
load_checkpoint_weight_patches,
)
__all__ = [
"SparseWeightPatch",
"SparseNCCLTrainerInitInfo",
"SparseNCCLWeightTransferUpdateInfo",
"SparseNCCLWeightTransferEngine",
"SparseNCCLTrainerWeightTransferEngine",
]
@dataclass
class SparseWeightPatch:
"""A sparse patch in checkpoint coordinates."""
name: str
indices: torch.Tensor
values: torch.Tensor
full_shape: tuple[int, ...]
"""Full checkpoint shape."""
@dataclass
class SparseNCCLTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for the sparse NCCL weight transfer backend.
Same rendezvous shape as the dense NCCL backend (the sender opens its
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
are never packed. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "sparse_nccl"
master_address: str
master_port: int
world_size: int
@dataclass
class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Update info for the sparse NCCL weight transfer backend."""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
num_updates_list: list[int]
"""Number of sparse entries to receive for each parameter in ``names``."""
def __post_init__(self) -> None:
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
f"`dtype_names` should be of the same size as `names`: "
f"got {len(self.dtype_names)} and {len(self.names)}"
)
if len(self.shapes) != num_params:
raise ValueError(
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
if len(self.num_updates_list) == 0:
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
if len(self.num_updates_list) != num_params:
raise ValueError(
f"`num_updates_list` should be of the same size as `names`: "
f"got {len(self.num_updates_list)} and {len(self.names)}"
)
if any(num_updates < 0 for num_updates in self.num_updates_list):
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
class SparseNCCLWeightTransferEngine(
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
):
"""Sparse weight transfer engine using NCCL.
Receives checkpoint-coordinate patches broadcast from the trainer and applies
them through the model's native weight loader. Sparse updates modify initialized
model tensors in place, so the layerwise reload lifecycle is not used.
"""
init_info_cls = NCCLWeightTransferInitInfo
update_info_cls = SparseNCCLWeightTransferUpdateInfo
supports_draft_weight_update = False
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
"""Initialize the NCCL process group with the trainer."""
self.model_update_group = worker_init_process_group(
init_info, self.parallel_config
)
def start_weight_update(self) -> None:
"""No-op: sparse patches are applied in place, no layerwise reload."""
pass
def finish_weight_update(self) -> None:
"""No-op: sparse patches are applied in place, no layerwise reload."""
pass
def receive_weights(self, update_info: SparseNCCLWeightTransferUpdateInfo) -> None:
"""Receive sparse flat-index patches from the trainer and apply them."""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
# Receive on the communicator's device rather than relying on the
# ambient current device in the RPC thread.
device = self.model_update_group.device
with torch.accelerator.device_index(device.index):
stream = torch.cuda.current_stream(device=device)
patches = []
for name, dtype_name, shape, num_updates in zip(
update_info.names,
update_info.dtype_names,
update_info.shapes,
update_info.num_updates_list,
strict=True,
):
dtype = getattr(torch, dtype_name)
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
values = torch.empty(num_updates, dtype=dtype, device=device)
self.model_update_group.broadcast(indices, src=0, stream=stream)
self.model_update_group.broadcast(values, src=0, stream=stream)
patches.append(
CheckpointWeightPatch(
name=name,
shape=tuple(shape),
dtype=dtype,
values=values,
indices=indices,
)
)
load_checkpoint_weight_patches(self.model, patches)
def shutdown(self) -> None:
if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None
class SparseNCCLTrainerWeightTransferEngine(
TrainerWeightTransferEngine[SparseNCCLTrainerInitInfo]
):
"""Trainer-side sparse NCCL weight transfer engine.
Broadcasts flat-index (indices, values) patches from NCCL rank 0 while the
inference-side `update_weights` runs concurrently on a side thread (the
worker's recvs rendezvous inside the same NCCL broadcasts). `send_weights`
owns a complete one-shot lifecycle. RL infrastructure that owns the generic
client lifecycle can call `send_weight_chunk` between one start and finish.
Sparse patches differ every round, so they are not a stable `WeightSource`:
the engine takes no `source`, and patches are passed directly to the send
methods. An empty patch list is a no-op.
Only the designated trainer sender joins the transfer group; other trainer
ranks skip sparse sends.
"""
init_info_cls = SparseNCCLTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
is_sender: bool = True,
) -> None:
# Sparse is a delta backend: it takes per-round patches via
# send_weights, so a `source` would silently never be sent. The
# parameter exists only to match the base/factory signature.
if source is not None:
raise ValueError(
"Sparse NCCL weight transfer takes no WeightSource; pass each "
"round's patches to send_weights(patches) instead."
)
super().__init__(client=client, source=source, is_sender=is_sender)
self.model_update_group: PyNcclCommunicator | None = None
@classmethod
def trainer_init(
cls,
init_info: SparseNCCLTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
engine = cls(client=client, source=source, is_sender=init_info.is_sender)
if not engine.is_sender:
return engine
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
# Sparse transfers are never packed, so the worker keeps the unpacked
# defaults on its init info.
worker_init_info = NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
rank_offset=1,
world_size=init_info.world_size,
)
# The inference workers block inside init_weight_transfer_engine waiting
# for the NCCL rendezvous, so we kick that off on a side thread while we
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine,
worker_init_payload(worker_init_info),
)
# Open the trainer endpoint as NCCL rank 0 on the current device
# (the init info satisfies the helper's rendezvous protocol).
engine.model_update_group = open_trainer_endpoint(init_info)
future.result() # surface any inference-side init error
return engine
def send_weights(self, patches: Iterable[SparseWeightPatch] | None = None) -> None:
"""Broadcast one sparse update through a one-shot lifecycle."""
patches = self._prepare_patches(patches)
if not patches:
return
self.client.start_weight_update()
self._broadcast_chunk(patches)
self.client.finish_weight_update()
def send_weight_chunk(
self, patches: Iterable[SparseWeightPatch] | None = None
) -> None:
"""Broadcast one chunk inside a caller-owned weight update lifecycle."""
patches = self._prepare_patches(patches)
if not patches:
return
self._broadcast_chunk(patches)
def _prepare_patches(
self, patches: Iterable[SparseWeightPatch] | None
) -> list[SparseWeightPatch]:
if not self.is_sender:
return []
prepared = list(patches) if patches is not None else []
if not prepared:
return []
if self.model_update_group is None:
raise RuntimeError("trainer_init() must be called before sending weights.")
for patch in prepared:
self._validate_patch(patch)
return prepared
def _broadcast_chunk(self, patches: list[SparseWeightPatch]) -> None:
assert self.model_update_group is not None
device = self.model_update_group.device
device_context = (
torch.accelerator.device_index(device.index)
if device.type != "cpu"
else nullcontext()
)
with device_context:
update_info = SparseNCCLWeightTransferUpdateInfo(
names=[patch.name for patch in patches],
dtype_names=[
str(patch.values.dtype).split(".")[-1] for patch in patches
],
shapes=[list(patch.full_shape) for patch in patches],
num_updates_list=[patch.indices.numel() for patch in patches],
)
# update_weights (workers receive) must run concurrently with the
# trainer-side broadcasts — both rendezvous inside the same NCCL calls.
executor = ThreadPoolExecutor(max_workers=1)
try:
future = executor.submit(
self.client.update_weights, asdict(update_info)
)
# Surface an RPC that failed before the worker entered NCCL rather
# than broadcasting to a peer that will never receive.
if future.done():
future.result()
stream = torch.cuda.current_stream()
for patch in patches:
self.model_update_group.broadcast(
patch.indices, src=0, stream=stream
)
self.model_update_group.broadcast(
patch.values, src=0, stream=stream
)
future.result() # surface inference-side errors
finally:
# A failed broadcast can leave the RPC blocked in the matching
# receive. Waiting for that thread would hide the original error.
executor.shutdown(wait=False, cancel_futures=True)
self._post_send_sync()
@staticmethod
def _validate_patch(patch: SparseWeightPatch) -> None:
"""Reject a malformed patch before starting the NCCL transfer."""
if patch.full_shape is None:
raise ValueError(f"Sparse patch requires full_shape: {patch.name}")
if patch.indices.dtype != torch.int32:
raise ValueError(
f"Sparse weight updates require int32 indices: {patch.name}"
)
if patch.indices.ndim != 1 or patch.values.ndim != 1:
raise ValueError(
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
)
if patch.indices.numel() != patch.values.numel():
raise ValueError(
f"`indices` and `values` must have matching lengths for {patch.name}"
)
def _post_send_sync(self) -> None:
"""Wait for the broadcasts to land before returning, so a caller may
rebuild or free the patch tensors as soon as a send method returns rather
than relying on same-stream ordering. See
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
cross-rank barrier."""
if torch.cuda.is_available():
torch.cuda.current_stream().synchronize()
def shutdown(self) -> None:
if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None