[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -77,21 +77,3 @@ class ECConnectorFactory:
|
||||
# Register various connectors here.
|
||||
# The registration should not be done in each individual file, as we want to
|
||||
# only load the files corresponding to the current connector.
|
||||
|
||||
ECConnectorFactory.register_connector(
|
||||
"ECExampleConnector",
|
||||
"vllm.distributed.ec_transfer.ec_connector.example_connector",
|
||||
"ECExampleConnector",
|
||||
)
|
||||
|
||||
ECConnectorFactory.register_connector(
|
||||
"ECCPUConnector",
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.connector",
|
||||
"ECCPUConnector",
|
||||
)
|
||||
|
||||
ECConnectorFactory.register_connector(
|
||||
"ECMooncakeConnector",
|
||||
"vllm.distributed.ec_transfer.ec_connector.mooncake_ec_connector",
|
||||
"ECMooncakeConnector",
|
||||
)
|
||||
|
||||
@@ -150,48 +150,12 @@ class KVConnectorFactory:
|
||||
# The registration should not be done in each individual file, as we want to
|
||||
# only load the files corresponding to the current connector.
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"ExampleHiddenStatesConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector",
|
||||
"ExampleHiddenStatesConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"NixlConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
|
||||
"NixlConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"NixlPullConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
|
||||
"NixlPullConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"NixlPushConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
|
||||
"NixlPushConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"MultiConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.multi_connector",
|
||||
"MultiConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"HiSparseConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector",
|
||||
"HiSparseConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"MoRIIOConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector",
|
||||
"MoRIIOConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"OffloadingConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector",
|
||||
@@ -203,14 +167,3 @@ KVConnectorFactory.register_connector(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector",
|
||||
"DecodeBenchConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"SimpleCPUOffloadConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector",
|
||||
"SimpleCPUOffloadConnector",
|
||||
)
|
||||
KVConnectorFactory.register_connector(
|
||||
"HF3FSKVConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_connector",
|
||||
"HF3FSKVConnector",
|
||||
)
|
||||
|
||||
@@ -1,302 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import threading
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import torch.utils.cpp_extension
|
||||
from torch.utils.cpp_extension import load
|
||||
|
||||
root = Path(__file__).parent.resolve()
|
||||
cuda_include_path = os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")
|
||||
hf3fs_utils = load(
|
||||
name="hf3fs_utils",
|
||||
sources=[f"{root}/utils/hf3fs_utils.cpp"],
|
||||
extra_include_paths=[cuda_include_path],
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HF3FS_AVAILABLE = True
|
||||
try:
|
||||
from hf3fs_fuse.io import (
|
||||
deregister_fd,
|
||||
extract_mount_point,
|
||||
make_ioring,
|
||||
make_iovec,
|
||||
register_fd,
|
||||
)
|
||||
except ImportError:
|
||||
HF3FS_AVAILABLE = False
|
||||
|
||||
|
||||
def rsynchronized():
|
||||
def _decorator(func):
|
||||
@wraps(func)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
with self.rlock:
|
||||
return func(self, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return _decorator
|
||||
|
||||
|
||||
def wsynchronized():
|
||||
def _decorator(func):
|
||||
@wraps(func)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
with self.wlock:
|
||||
return func(self, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return _decorator
|
||||
|
||||
|
||||
class Hf3fsClient:
|
||||
def __init__(self, path: str, size: int, bytes_per_page: int, entries: int):
|
||||
"""Initialize the HF3FS client with hf3fs_fuse.
|
||||
|
||||
Args:
|
||||
path: Path to the file used for storage
|
||||
size: Total size of the storage file in bytes
|
||||
bytes_per_page: Size of each page in bytes
|
||||
entries: Maximum number of concurrent operations
|
||||
|
||||
"""
|
||||
if not HF3FS_AVAILABLE:
|
||||
raise ImportError(
|
||||
"hf3fs_fuse.io is not available. Please install the hf3fs_fuse package."
|
||||
)
|
||||
|
||||
self.path = path
|
||||
self.size = size
|
||||
self.bytes_per_page = bytes_per_page
|
||||
self.entries = entries
|
||||
|
||||
self._closed = False
|
||||
|
||||
self.file = None
|
||||
self.shm_r = None
|
||||
self.shm_w = None
|
||||
self.ior_r = None
|
||||
self.ior_w = None
|
||||
self.iov_r = None
|
||||
self.iov_w = None
|
||||
try:
|
||||
# Create the file if it doesn't exist and set its size
|
||||
self.file = os.open(self.path, os.O_RDWR | os.O_CREAT)
|
||||
os.ftruncate(self.file, size)
|
||||
register_fd(self.file)
|
||||
|
||||
self.hf3fs_mount_point = extract_mount_point(path)
|
||||
self.bs = self.bytes_per_page
|
||||
self.shm_r = multiprocessing.shared_memory.SharedMemory(
|
||||
size=self.bs * self.entries, create=True
|
||||
)
|
||||
self.shm_w = multiprocessing.shared_memory.SharedMemory(
|
||||
size=self.bs * self.entries, create=True
|
||||
)
|
||||
|
||||
self.shm_r_tensor = torch.frombuffer(self.shm_r.buf, dtype=torch.uint8)
|
||||
self.shm_w_tensor = torch.frombuffer(self.shm_w.buf, dtype=torch.uint8)
|
||||
|
||||
numel = self.bs * self.entries
|
||||
self.r_pinned = torch.empty(
|
||||
numel,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
)
|
||||
self.w_pinned = torch.empty(
|
||||
numel,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
)
|
||||
|
||||
self.numa = -1
|
||||
self.ior_r = make_ioring(
|
||||
self.hf3fs_mount_point,
|
||||
self.entries,
|
||||
for_read=True,
|
||||
timeout=1,
|
||||
numa=self.numa,
|
||||
)
|
||||
self.ior_w = make_ioring(
|
||||
self.hf3fs_mount_point,
|
||||
self.entries,
|
||||
for_read=False,
|
||||
timeout=1,
|
||||
numa=self.numa,
|
||||
)
|
||||
self.iov_r = make_iovec(self.shm_r, self.hf3fs_mount_point)
|
||||
self.iov_w = make_iovec(self.shm_w, self.hf3fs_mount_point)
|
||||
self.shm_r.unlink()
|
||||
self.shm_w.unlink()
|
||||
|
||||
self.rlock = threading.RLock()
|
||||
self.wlock = threading.RLock()
|
||||
|
||||
self.stream = torch.cuda.Stream()
|
||||
self.stream_ptr_int = self.stream.cuda_stream
|
||||
|
||||
except Exception:
|
||||
self._release_resources()
|
||||
raise
|
||||
|
||||
logger.debug(
|
||||
"Initialized HF3FS client with file: %s, size: %s bytes", path, size
|
||||
)
|
||||
|
||||
def _release_resources(self) -> None:
|
||||
"""Release all acquired resources safely."""
|
||||
# iov must be released before ioring and shm
|
||||
for attr in ("iov_r", "iov_w", "ior_r", "ior_w"):
|
||||
obj = getattr(self, attr, None)
|
||||
if obj is not None:
|
||||
del obj
|
||||
setattr(self, attr, None)
|
||||
|
||||
for attr in ("shm_r", "shm_w"):
|
||||
shm = getattr(self, attr, None)
|
||||
if shm is not None:
|
||||
try:
|
||||
shm.close()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to close %s: %s", attr, e)
|
||||
setattr(self, attr, None)
|
||||
|
||||
if self.file is not None:
|
||||
try:
|
||||
deregister_fd(self.file)
|
||||
except Exception as e:
|
||||
logger.warning("deregister_fd failed: %s", e)
|
||||
try:
|
||||
os.close(self.file)
|
||||
except OSError as e:
|
||||
logger.warning("os.close failed: %s", e)
|
||||
self.file = None
|
||||
|
||||
@rsynchronized()
|
||||
def batch_read(self, offsets: list[int], tensors: list[torch.Tensor]) -> list[int]:
|
||||
"""Read data from the file at specified offsets into tensors.
|
||||
|
||||
Args:
|
||||
offsets: List of byte offsets to read from
|
||||
tensors: List of tensors to read data into
|
||||
|
||||
Returns:
|
||||
List of operation results (0 for success, non-zero for error)
|
||||
|
||||
"""
|
||||
self.check(offsets, tensors)
|
||||
assert self.ior_r is not None
|
||||
assert self.iov_r is not None
|
||||
|
||||
# prepare
|
||||
current = 0
|
||||
for offset, tensor in zip(offsets, tensors):
|
||||
size = tensor.numel() * tensor.itemsize
|
||||
self.ior_r.prepare(
|
||||
self.iov_r[current : current + size], True, self.file, offset
|
||||
)
|
||||
current += size
|
||||
|
||||
# submit
|
||||
ionum = len(offsets)
|
||||
resv = self.ior_r.submit().wait(min_results=ionum)
|
||||
|
||||
# results
|
||||
with torch.cuda.stream(self.stream):
|
||||
hf3fs_utils.read_shm(
|
||||
self.shm_r_tensor, self.r_pinned, tensors, self.stream_ptr_int
|
||||
)
|
||||
results = [res.result for res in resv]
|
||||
|
||||
return results
|
||||
|
||||
@wsynchronized()
|
||||
def batch_write(
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
|
||||
) -> list[int]:
|
||||
"""Write data from tensors to the file at specified offsets.
|
||||
|
||||
Args:
|
||||
offsets: List of byte offsets to write to
|
||||
tensors: List of tensors containing data to write
|
||||
event: CUDA event to synchronize on before reading the tensors
|
||||
|
||||
Returns:
|
||||
List of operation results (0 for success, non-zero for error)
|
||||
|
||||
"""
|
||||
self.check(offsets, tensors)
|
||||
assert self.ior_w is not None
|
||||
assert self.iov_w is not None
|
||||
|
||||
# prepare
|
||||
with torch.cuda.stream(self.stream):
|
||||
self.stream.wait_event(event)
|
||||
hf3fs_utils.write_shm(
|
||||
tensors, self.shm_w_tensor, self.w_pinned, self.stream_ptr_int
|
||||
)
|
||||
|
||||
current = 0
|
||||
for offset, tensor in zip(offsets, tensors):
|
||||
size = tensor.numel() * tensor.itemsize
|
||||
self.ior_w.prepare(
|
||||
self.iov_w[current : current + size], False, self.file, offset
|
||||
)
|
||||
current += size
|
||||
|
||||
# submit
|
||||
ionum = len(offsets)
|
||||
resv = self.ior_w.submit().wait(min_results=ionum)
|
||||
|
||||
# results
|
||||
results = [res.result for res in resv]
|
||||
|
||||
return results
|
||||
|
||||
def check(self, offsets: list[int], tensors: list[torch.Tensor]) -> None:
|
||||
sizes = [t.numel() * t.itemsize for t in tensors]
|
||||
if any(
|
||||
[
|
||||
len(offsets) > self.entries,
|
||||
len(offsets) != len(sizes),
|
||||
any(
|
||||
offset < 0 or offset + size > self.size
|
||||
for offset, size in zip(offsets, sizes)
|
||||
),
|
||||
any(size > self.bytes_per_page for size in sizes),
|
||||
]
|
||||
):
|
||||
self.close()
|
||||
raise ValueError("Hf3fsClient.check Failed")
|
||||
|
||||
def get_size(self) -> int:
|
||||
"""Get the total size of the storage file.
|
||||
|
||||
Returns:
|
||||
Size of the file in bytes
|
||||
|
||||
"""
|
||||
return self.size
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the client and clean up resources."""
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
self._release_resources()
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Flush any pending writes to disk."""
|
||||
if not self._closed and self.file is not None:
|
||||
os.fsync(self.file)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,532 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HF3FS Metadata Server with key-based organization."""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import threading
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
|
||||
try:
|
||||
import orjson
|
||||
|
||||
HAS_ORJSON = True
|
||||
except ImportError:
|
||||
import json as orjson # type: ignore
|
||||
|
||||
HAS_ORJSON = False
|
||||
|
||||
import requests
|
||||
from fastapi import FastAPI, HTTPException, Request, Response
|
||||
from fastapi.responses import ORJSONResponse
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RankFileMetadata:
|
||||
"""Manages file page allocation for a single rank."""
|
||||
|
||||
rank_id: int
|
||||
num_pages: int
|
||||
free_pages: list[int]
|
||||
|
||||
def allocate_pages(self, num_pages: int) -> list[int]:
|
||||
"""Allocate specified number of free pages."""
|
||||
if len(self.free_pages) < num_pages:
|
||||
return []
|
||||
|
||||
allocated = self.free_pages[:num_pages]
|
||||
self.free_pages = self.free_pages[num_pages:]
|
||||
return allocated
|
||||
|
||||
def release_pages(self, page_indices: list[int]) -> None:
|
||||
"""Release pages back to free pool."""
|
||||
for page_idx in page_indices:
|
||||
if page_idx not in self.free_pages:
|
||||
self.free_pages.append(page_idx)
|
||||
|
||||
def get_free_page_count(self) -> int:
|
||||
"""Get current number of free pages."""
|
||||
return len(self.free_pages)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KeyMetadata:
|
||||
"""Manages metadata for a single key across multiple ranks."""
|
||||
|
||||
key: str
|
||||
rank_to_page: dict[int, int] # rank -> allocated page index
|
||||
tp_world_size: int
|
||||
|
||||
def add_rank_page(self, rank: int, page_index: int) -> None:
|
||||
"""Add page allocation for a specific rank."""
|
||||
self.rank_to_page[rank] = page_index
|
||||
|
||||
def get_all_pages(self) -> list[tuple[int, int]]:
|
||||
"""Get all (rank, page) pairs for this key."""
|
||||
return [(rank, page) for rank, page in self.rank_to_page.items()]
|
||||
|
||||
def get_rank_page(self, rank: int) -> int | None:
|
||||
"""Get page index for a specific rank."""
|
||||
return self.rank_to_page.get(rank)
|
||||
|
||||
def is_complete(self) -> bool:
|
||||
"""Check if all ranks in the TP world have allocated pages."""
|
||||
return len(self.rank_to_page) == self.tp_world_size
|
||||
|
||||
|
||||
class GlobalMetadataState:
|
||||
"""Manages global metadata state across all ranks and keys."""
|
||||
|
||||
def __init__(self):
|
||||
self.global_lock = threading.RLock()
|
||||
self.rank_metadata: dict[int, RankFileMetadata] = {}
|
||||
self.key_metadata: dict[str, KeyMetadata] = {}
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all metadata state."""
|
||||
with self.global_lock:
|
||||
self.rank_metadata.clear()
|
||||
self.key_metadata.clear()
|
||||
logger.info("Cleared all metadata state")
|
||||
|
||||
def initialize_rank(self, rank: int, num_pages: int) -> None:
|
||||
"""Initialize a new rank with specified number of pages."""
|
||||
with self.global_lock:
|
||||
if rank not in self.rank_metadata:
|
||||
self.rank_metadata[rank] = RankFileMetadata(
|
||||
rank, num_pages, list(range(num_pages))
|
||||
)
|
||||
logger.info("Initialized rank %s with %s pages", rank, num_pages)
|
||||
|
||||
def allocate_pages_for_keys(
|
||||
self, rank: int, keys: list[tuple[str, str]]
|
||||
) -> dict[str, int]:
|
||||
"""Allocate one page for each key on the specified rank.
|
||||
|
||||
Args:
|
||||
rank: Rank ID to allocate pages on
|
||||
keys: List of keys to allocate pages for
|
||||
|
||||
Returns:
|
||||
Dictionary mapping key -> allocated page index
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
if rank not in self.rank_metadata:
|
||||
raise ValueError(f"Rank {rank} not initialized")
|
||||
|
||||
# Batch allocate pages for all keys
|
||||
num_pages_needed = len(keys)
|
||||
allocated_pages = self.rank_metadata[rank].allocate_pages(num_pages_needed)
|
||||
|
||||
if len(allocated_pages) < num_pages_needed:
|
||||
logger.warning(
|
||||
"Rank %s only allocated %s pages for %s keys",
|
||||
rank,
|
||||
len(allocated_pages),
|
||||
num_pages_needed,
|
||||
)
|
||||
|
||||
allocation_results = {}
|
||||
for i, (key, prefix_key) in enumerate(keys):
|
||||
if key in self.key_metadata:
|
||||
key_meta = self.key_metadata[key]
|
||||
if key_meta.is_complete() and rank in key_meta.rank_to_page:
|
||||
# key is already fully written, reuse the existing page
|
||||
# and release the allocated pages back to the free pool.
|
||||
if i < len(allocated_pages):
|
||||
self.rank_metadata[rank].release_pages([allocated_pages[i]])
|
||||
allocation_results[key] = key_meta.rank_to_page[rank]
|
||||
continue
|
||||
|
||||
if i < len(allocated_pages):
|
||||
allocation_results[key] = allocated_pages[i]
|
||||
else:
|
||||
allocation_results[key] = -1 # No pages available
|
||||
|
||||
return allocation_results
|
||||
|
||||
def confirm_write_for_keys(
|
||||
self,
|
||||
rank: int,
|
||||
key_confirmations: list[tuple[str, int]],
|
||||
pages_to_release: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Confirm write operations for keys and update metadata.
|
||||
|
||||
Args:
|
||||
rank: Rank ID that confirmed the writes
|
||||
key_confirmations: List of (key, page_index) tuples
|
||||
pages_to_release: List of page indices to release back to free pool
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
# Confirm successful writes
|
||||
for key, page_index in key_confirmations:
|
||||
if key not in self.key_metadata:
|
||||
# Need to determine tp_world_size from rank_metadata
|
||||
tp_world_size = len(self.rank_metadata)
|
||||
self.key_metadata[key] = KeyMetadata(key, {}, tp_world_size)
|
||||
|
||||
# Add confirmed page to key metadata
|
||||
self.key_metadata[key].add_rank_page(rank, page_index)
|
||||
|
||||
# Release specified pages back to free pool
|
||||
if pages_to_release:
|
||||
self.rank_metadata[rank].release_pages(pages_to_release)
|
||||
logger.debug(
|
||||
"Released %s pages on rank %s: %s",
|
||||
len(pages_to_release),
|
||||
rank,
|
||||
pages_to_release,
|
||||
)
|
||||
|
||||
def batch_key_exists(self, keys: list[str]) -> list[bool]:
|
||||
"""Check if keys exist in metadata and all ranks have confirmed writes.
|
||||
|
||||
Args:
|
||||
keys: List of keys to check
|
||||
|
||||
Returns:
|
||||
List of boolean values indicating key existence and completion
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
results = []
|
||||
for key in keys:
|
||||
if key not in self.key_metadata:
|
||||
results.append(False)
|
||||
else:
|
||||
# Check if all ranks in the TP world have confirmed writes
|
||||
key_meta = self.key_metadata[key]
|
||||
results.append(key_meta.is_complete())
|
||||
return results
|
||||
|
||||
def get_key_locations(self, rank: int, keys: list[str]) -> list[int | None]:
|
||||
"""Get page indices for keys on a specific rank.
|
||||
|
||||
Args:
|
||||
rank: Rank ID to query
|
||||
keys: List of keys to look up
|
||||
|
||||
Returns:
|
||||
List of page indices in the same order as input keys (None if key not found)
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
if rank not in self.rank_metadata:
|
||||
raise ValueError(f"Rank {rank} not initialized")
|
||||
|
||||
results = []
|
||||
for key in keys:
|
||||
if key in self.key_metadata:
|
||||
key_meta = self.key_metadata[key]
|
||||
if key_meta.is_complete():
|
||||
page_index = key_meta.get_rank_page(rank)
|
||||
else:
|
||||
page_index = None
|
||||
|
||||
results.append(page_index)
|
||||
else:
|
||||
results.append(None)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class Hf3fsMetadataServer:
|
||||
"""HF3FS Metadata Server with improved key-based organization."""
|
||||
|
||||
def __init__(self, persistence_path: str | None = None, save_interval: int = 60):
|
||||
self.state = GlobalMetadataState()
|
||||
if HAS_ORJSON:
|
||||
self.app = FastAPI(default_response_class=ORJSONResponse)
|
||||
else:
|
||||
self.app = FastAPI()
|
||||
self._setup_routes()
|
||||
|
||||
async def _read_json(self, request: Request) -> dict:
|
||||
"""Parse request JSON using orjson if available."""
|
||||
body = await request.body()
|
||||
return orjson.loads(body)
|
||||
|
||||
def _json_response(self, content: dict):
|
||||
"""Return ORJSONResponse when available to bypass jsonable_encoder."""
|
||||
if HAS_ORJSON:
|
||||
return ORJSONResponse(content)
|
||||
else:
|
||||
return content
|
||||
|
||||
def _setup_routes(self):
|
||||
"""Setup FastAPI routes for new API design."""
|
||||
self.app.post("/rank/{rank}/initialize")(self.initialize_rank)
|
||||
self.app.post("/keys/batch_allocate")(self.batch_allocate_pages_for_keys)
|
||||
self.app.post("/keys/confirm_write")(self.confirm_write_for_keys)
|
||||
self.app.post("/keys/batch_exists")(self.batch_key_exists)
|
||||
self.app.post("/keys/get_locations")(self.get_key_locations)
|
||||
self.app.post("/clear")(self.clear)
|
||||
|
||||
async def initialize_rank(self, rank: int, request: Request):
|
||||
"""Initialize a rank with specified number of pages."""
|
||||
data = await self._read_json(request)
|
||||
role = data.get("role", "worker")
|
||||
num_pages = data.get("num_pages", 0)
|
||||
|
||||
if role == "scheduler":
|
||||
return self._json_response(
|
||||
{"message": "Scheduler role does not require initialization"}
|
||||
)
|
||||
|
||||
if role == "worker" and num_pages > 0:
|
||||
self.state.initialize_rank(rank, num_pages)
|
||||
return self._json_response(
|
||||
{"message": f"Rank {rank} initialized with {num_pages} pages"}
|
||||
)
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Invalid initialization parameters"
|
||||
)
|
||||
|
||||
async def batch_allocate_pages_for_keys(self, request: Request):
|
||||
"""Allocate one page for each key on a specific rank."""
|
||||
data = await self._read_json(request)
|
||||
rank = data.get("rank")
|
||||
keys = data.get("keys", [])
|
||||
|
||||
# Validate input format
|
||||
if rank is None or not isinstance(keys, list):
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Invalid request format: need 'rank' and 'keys'"
|
||||
)
|
||||
|
||||
try:
|
||||
# Perform allocation
|
||||
results = self.state.allocate_pages_for_keys(rank, keys)
|
||||
|
||||
# Convert results to response format
|
||||
response = {"rank": rank, "results": list(results.items())}
|
||||
return self._json_response(response)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Allocation failed: {str(e)}"
|
||||
) from e
|
||||
|
||||
async def confirm_write_for_keys(self, request: Request):
|
||||
"""Confirm write operations for keys."""
|
||||
data = await self._read_json(request)
|
||||
rank = data.get("rank")
|
||||
confirmations = data.get("confirmations", [])
|
||||
pages_to_release = data.get("pages_to_release", [])
|
||||
|
||||
# Validate input format
|
||||
if rank is None or not isinstance(confirmations, list):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Invalid request format: need 'rank' and 'confirmations'",
|
||||
)
|
||||
|
||||
try:
|
||||
self.state.confirm_write_for_keys(rank, confirmations, pages_to_release)
|
||||
|
||||
return Response(status_code=204)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Confirm write for keys failed: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Confirmation failed: {str(e)}"
|
||||
) from e
|
||||
|
||||
async def batch_key_exists(self, request: Request):
|
||||
"""Check if multiple keys exist in metadata."""
|
||||
data = await self._read_json(request)
|
||||
keys = data.get("keys", [])
|
||||
|
||||
if not isinstance(keys, list):
|
||||
raise HTTPException(status_code=400, detail="Invalid keys format")
|
||||
|
||||
try:
|
||||
exists_results = self.state.batch_key_exists(keys)
|
||||
return self._json_response({"exists": exists_results})
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Key existence check failed: {str(e)}"
|
||||
) from e
|
||||
|
||||
async def get_key_locations(self, request: Request):
|
||||
"""Get page indices for keys on a specific rank."""
|
||||
data = await self._read_json(request)
|
||||
rank = data.get("rank")
|
||||
keys = data.get("keys", [])
|
||||
|
||||
# Validate input format
|
||||
if rank is None or not isinstance(keys, list):
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Invalid request format: need 'rank' and 'keys'"
|
||||
)
|
||||
|
||||
try:
|
||||
# Get key locations
|
||||
locations = self.state.get_key_locations(rank, keys)
|
||||
return self._json_response({"locations": locations})
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Failed to get key locations: {str(e)}"
|
||||
) from e
|
||||
|
||||
async def clear(self, request: Request):
|
||||
"""Clear the metadata server."""
|
||||
self.state.clear()
|
||||
return Response(status_code=204)
|
||||
|
||||
def run(self, host: str = "0.0.0.0", port: int = 18000):
|
||||
"""Run the metadata server."""
|
||||
import uvicorn
|
||||
|
||||
logger.info("Starting improved metadata server on http://%s:%s", host, port)
|
||||
uvicorn.run(self.app, host=host, port=port)
|
||||
|
||||
|
||||
# --- Client implementation ---
|
||||
class Hf3fsMetadataInterface(ABC):
|
||||
"""Interface for HF3FS metadata operations."""
|
||||
|
||||
@abstractmethod
|
||||
def initialize(self, rank: int, num_pages: int = 0, role: str = "worker") -> None:
|
||||
"""Initialize the metadata service with specified number of pages."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def allocate_pages_for_keys(
|
||||
self, rank: int, keys: list[tuple[str, str]]
|
||||
) -> list[tuple[str, int]]:
|
||||
"""Allocate one page for each key on the specified rank."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def confirm_write_for_keys(
|
||||
self,
|
||||
rank: int,
|
||||
key_confirmations: list[tuple[str, int]],
|
||||
pages_to_release: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Confirm write operations for keys and optionally release pages."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def batch_key_exists(self, keys: list[str]) -> list[bool]:
|
||||
"""Check if keys exist and are complete across all ranks."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_key_locations(self, rank: int, keys: list[str]) -> list[int]:
|
||||
"""Get page indices for keys on a specific rank."""
|
||||
pass
|
||||
|
||||
|
||||
class Hf3fsGlobalMetadataClient(Hf3fsMetadataInterface):
|
||||
"""Global HTTP metadata client for HF3FS."""
|
||||
|
||||
def __init__(self, base_url: str = "http://localhost:18000", max_retries: int = 3):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self._session = requests.Session()
|
||||
|
||||
retry_strategy = Retry(
|
||||
total=max_retries,
|
||||
backoff_factor=0.3,
|
||||
status_forcelist=[500, 502, 503, 504],
|
||||
allowed_methods=["GET", "POST"],
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry_strategy)
|
||||
self._session.mount("http://", adapter)
|
||||
|
||||
def _post(self, endpoint: str, json_data: dict) -> dict:
|
||||
"""Make POST request to metadata server."""
|
||||
try:
|
||||
url = f"{self.base_url}/{endpoint}"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if HAS_ORJSON:
|
||||
payload = orjson.dumps(json_data)
|
||||
else:
|
||||
import json
|
||||
|
||||
payload = json.dumps(json_data).encode("utf-8")
|
||||
response = self._session.post(url, data=payload, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
if response.status_code == 204 or not response.content:
|
||||
return {}
|
||||
if HAS_ORJSON:
|
||||
return orjson.loads(response.content)
|
||||
else:
|
||||
return response.json()
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error("Failed to POST to %s after retries: %s", endpoint, e)
|
||||
raise RuntimeError(f"Failed to connect to metadata server: {e}") from e
|
||||
|
||||
def initialize(self, rank: int, num_pages: int = 0, role: str = "worker") -> None:
|
||||
"""Initialize a rank with specified number of pages."""
|
||||
self._post(f"rank/{rank}/initialize", {"num_pages": num_pages, "role": role})
|
||||
|
||||
def allocate_pages_for_keys(
|
||||
self, rank: int, keys: list[tuple[str, str]]
|
||||
) -> list[tuple[str, int]]:
|
||||
"""Allocate pages for keys on the specified rank."""
|
||||
response = self._post("keys/batch_allocate", {"rank": rank, "keys": keys})
|
||||
|
||||
# Convert response to expected format
|
||||
return response.get("results", {})
|
||||
|
||||
def confirm_write_for_keys(
|
||||
self,
|
||||
rank: int,
|
||||
key_confirmations: list[tuple[str, int]],
|
||||
pages_to_release: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Confirm write operations for keys and optionally release pages."""
|
||||
payload = {
|
||||
"rank": rank,
|
||||
"confirmations": key_confirmations,
|
||||
"pages_to_release": pages_to_release or [],
|
||||
}
|
||||
|
||||
self._post("keys/confirm_write", payload)
|
||||
|
||||
def batch_key_exists(self, keys: list[str]) -> list[bool]:
|
||||
"""Check if keys exist and are complete across all ranks."""
|
||||
response = self._post("keys/batch_exists", {"keys": keys})
|
||||
return response.get("exists", [])
|
||||
|
||||
def get_key_locations(self, rank: int, keys: list[str]) -> list[int]:
|
||||
"""Get page indices for keys on a specific rank."""
|
||||
response = self._post("keys/get_locations", {"rank": rank, "keys": keys})
|
||||
return response.get("locations", [])
|
||||
|
||||
|
||||
def run_metadata_server(
|
||||
host: str = "0.0.0.0",
|
||||
port: int = 18000,
|
||||
):
|
||||
"""Run the improved HF3FS metadata server."""
|
||||
server = Hf3fsMetadataServer()
|
||||
server.run(host=host, port=port)
|
||||
|
||||
|
||||
# --- Main Execution ---
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Improved HF3FS Metadata Server")
|
||||
parser.add_argument(
|
||||
"--host", type=str, default="0.0.0.0", help="Host to bind the server to."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=18000, help="Port to run the server on."
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
run_metadata_server(args.host, args.port)
|
||||
@@ -1,139 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
class AtomicCounter:
|
||||
"""Thread-safe atomic counter for round-robin operations."""
|
||||
|
||||
def __init__(self, n: int):
|
||||
assert n > 0, "Counter size must be positive"
|
||||
self._n = n
|
||||
self._value = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def next(self) -> int:
|
||||
"""Get next value in round-robin fashion."""
|
||||
with self._lock:
|
||||
current = self._value
|
||||
self._value = (current + 1) % self._n
|
||||
return current
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadBlockInfo:
|
||||
"""Operation for loading blocks from external storage."""
|
||||
|
||||
num_computed_blocks: int
|
||||
num_blocks_to_load: int
|
||||
need_fetch_block_ids: list[int]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SaveBlockInfo:
|
||||
"""Operation for saving blocks to external storage."""
|
||||
|
||||
skip_leading_blocks: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestSchedulingState:
|
||||
"""Unified request scheduling state management."""
|
||||
|
||||
request_id: str
|
||||
request: Request | None = None
|
||||
|
||||
# Token and block tracking
|
||||
token_ids: list[int] = field(default_factory=list)
|
||||
allocated_block_ids: list[int] = field(default_factory=list)
|
||||
num_saved_blocks: int = 0
|
||||
|
||||
# Load operation info
|
||||
load_op: LoadBlockInfo | None = None
|
||||
|
||||
# Scheduling phase
|
||||
phase: str = "NEW" # NEW -> WAITING_TO_LOAD -> ACTIVE -> FINISHED
|
||||
|
||||
def needs_loading(self) -> bool:
|
||||
"""Check if request needs loading."""
|
||||
return self.load_op is not None and self.load_op.num_blocks_to_load > 0
|
||||
|
||||
def is_ready_to_load(self) -> bool:
|
||||
"""Check if request is ready for loading."""
|
||||
return self.phase == "WAITING_TO_LOAD" and self.needs_loading()
|
||||
|
||||
def update_tokens_and_blocks(self, new_token_ids: list[int], new_block_ids) -> None:
|
||||
"""Update with new tokens and blocks."""
|
||||
if new_token_ids:
|
||||
self.token_ids.extend(new_token_ids)
|
||||
|
||||
if new_block_ids is not None:
|
||||
normalized_block_ids = self._normalize_block_ids(new_block_ids)
|
||||
self.allocated_block_ids.extend(normalized_block_ids)
|
||||
|
||||
def _normalize_block_ids(self, block_ids) -> list[int]:
|
||||
"""Normalize block_ids to list format."""
|
||||
if not block_ids:
|
||||
return []
|
||||
if isinstance(block_ids, tuple):
|
||||
return block_ids[0] if block_ids else []
|
||||
if isinstance(block_ids, list):
|
||||
return block_ids
|
||||
return []
|
||||
|
||||
|
||||
@dataclass
|
||||
class HF3FSRequestMetadata:
|
||||
"""Metadata for a single request in HF3FS connector."""
|
||||
|
||||
request_id: str
|
||||
token_ids: list[int]
|
||||
block_ids: list[int]
|
||||
load_block_op: LoadBlockInfo | None = None
|
||||
save_block_op: SaveBlockInfo | None = None
|
||||
|
||||
@staticmethod
|
||||
def from_scheduling_state(
|
||||
state: "RequestSchedulingState",
|
||||
block_size: int,
|
||||
load_op: LoadBlockInfo | None = None,
|
||||
skip_leading_blocks: int | None = None,
|
||||
) -> Optional["HF3FSRequestMetadata"]:
|
||||
"""Create request metadata from scheduling state."""
|
||||
token_count = len(state.token_ids)
|
||||
total_blocks = token_count // block_size
|
||||
|
||||
skip_blocks = (
|
||||
state.num_saved_blocks
|
||||
if skip_leading_blocks is None
|
||||
else skip_leading_blocks
|
||||
)
|
||||
|
||||
new_blocks_to_save = total_blocks - state.num_saved_blocks
|
||||
if new_blocks_to_save <= 0 and load_op is None:
|
||||
return None
|
||||
|
||||
state.num_saved_blocks = total_blocks
|
||||
return HF3FSRequestMetadata(
|
||||
request_id=state.request_id,
|
||||
token_ids=state.token_ids,
|
||||
block_ids=state.allocated_block_ids,
|
||||
load_block_op=load_op,
|
||||
save_block_op=SaveBlockInfo(skip_leading_blocks=skip_blocks),
|
||||
)
|
||||
|
||||
|
||||
class HF3FSConnectorMetadata(KVConnectorMetadata):
|
||||
"""Container for HF3FS connector metadata."""
|
||||
|
||||
def __init__(self):
|
||||
self.requests: list[HF3FSRequestMetadata] = []
|
||||
|
||||
def add_request(self, request_metadata: HF3FSRequestMetadata) -> None:
|
||||
"""Add request to metadata."""
|
||||
self.requests.append(request_metadata)
|
||||
@@ -1,262 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
@triton.jit
|
||||
def kv_cache_scatter_kernel(
|
||||
kv_cache_ptrs_ptr,
|
||||
source_ptr,
|
||||
token_indices_ptr,
|
||||
num_tokens_in_block,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
tokens_per_block,
|
||||
block_stride,
|
||||
head_stride,
|
||||
token_stride,
|
||||
content_stride,
|
||||
num_heads: tl.constexpr,
|
||||
content_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
layer_idx = tl.program_id(0)
|
||||
token_pos = tl.program_id(1)
|
||||
|
||||
if layer_idx >= num_layers or token_pos >= num_tokens_in_block:
|
||||
return
|
||||
|
||||
token_idx = tl.load(token_indices_ptr + token_pos)
|
||||
kv_cache_ptr = tl.cast(tl.load(kv_cache_ptrs_ptr + layer_idx), source_ptr.dtype)
|
||||
|
||||
if token_idx >= total_token_in_kvcache:
|
||||
return
|
||||
|
||||
block_idx = token_idx // tokens_per_block
|
||||
state_idx = token_idx % tokens_per_block
|
||||
hidden_size = num_heads * content_size
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < hidden_size
|
||||
head_idx = offset // content_size
|
||||
content_idx = offset % content_size
|
||||
source_offset = (
|
||||
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
|
||||
) * content_size + content_idx
|
||||
target_offset = (
|
||||
block_idx * block_stride
|
||||
+ head_idx * head_stride
|
||||
+ state_idx * token_stride
|
||||
+ content_idx * content_stride
|
||||
)
|
||||
value = tl.load(source_ptr + source_offset, mask=mask)
|
||||
tl.store(kv_cache_ptr + target_offset, value, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def kv_cache_gather_kernel(
|
||||
kv_cache_ptrs_ptr,
|
||||
dst_ptr,
|
||||
token_indices_ptr,
|
||||
num_tokens_in_block,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
tokens_per_block,
|
||||
block_stride,
|
||||
head_stride,
|
||||
token_stride,
|
||||
content_stride,
|
||||
num_heads: tl.constexpr,
|
||||
content_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
layer_idx = tl.program_id(0)
|
||||
token_pos = tl.program_id(1)
|
||||
|
||||
if layer_idx >= num_layers or token_pos >= num_tokens_in_block:
|
||||
return
|
||||
|
||||
token_idx = tl.load(token_indices_ptr + token_pos)
|
||||
kv_cache_ptr = tl.cast(tl.load(kv_cache_ptrs_ptr + layer_idx), dst_ptr.dtype)
|
||||
|
||||
if token_idx >= total_token_in_kvcache:
|
||||
return
|
||||
|
||||
block_idx = token_idx // tokens_per_block
|
||||
state_idx = token_idx % tokens_per_block
|
||||
hidden_size = num_heads * content_size
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < hidden_size
|
||||
head_idx = offset // content_size
|
||||
content_idx = offset % content_size
|
||||
kvcache_offset = (
|
||||
block_idx * block_stride
|
||||
+ head_idx * head_stride
|
||||
+ state_idx * token_stride
|
||||
+ content_idx * content_stride
|
||||
)
|
||||
dst_offset = (
|
||||
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
|
||||
) * content_size + content_idx
|
||||
value = tl.load(kv_cache_ptr + kvcache_offset, mask=mask)
|
||||
tl.store(dst_ptr + dst_offset, value, mask=mask)
|
||||
|
||||
|
||||
def scatter_kv_caches(
|
||||
kv_caches_ptrs: torch.Tensor,
|
||||
total_token_in_kvcache: int,
|
||||
src_tensor: torch.Tensor,
|
||||
token_indices: list[int],
|
||||
tokens_per_block: int,
|
||||
num_heads: int,
|
||||
content_size: int,
|
||||
kv_cache_strides: tuple[int, ...],
|
||||
) -> None:
|
||||
"""Scatter KV cache data from source tensor to KV cache storage.
|
||||
|
||||
Args:
|
||||
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
|
||||
total_token_in_kvcache: Total number of tokens in KV cache
|
||||
src_tensor: Source ``[L, H, N, C]`` tensor containing data to scatter
|
||||
token_indices: List of token positions to update
|
||||
tokens_per_block: Number of stored states in each cache block
|
||||
num_heads: Size of the H axis
|
||||
content_size: Size of the C axis
|
||||
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
|
||||
|
||||
"""
|
||||
num_layers = len(kv_caches_ptrs)
|
||||
num_tokens_in_block = len(token_indices)
|
||||
|
||||
assert src_tensor.shape == (
|
||||
num_layers,
|
||||
num_heads,
|
||||
num_tokens_in_block,
|
||||
content_size,
|
||||
)
|
||||
assert len(kv_cache_strides) == 4
|
||||
|
||||
device = src_tensor.device
|
||||
token_indices_tensor = async_tensor_h2d(
|
||||
token_indices, device=device, dtype=torch.int32
|
||||
)
|
||||
|
||||
grid = (num_layers, num_tokens_in_block)
|
||||
BLOCK_SIZE = 128
|
||||
|
||||
kv_cache_scatter_kernel[grid](
|
||||
kv_caches_ptrs,
|
||||
src_tensor,
|
||||
token_indices_tensor,
|
||||
num_tokens_in_block,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
tokens_per_block,
|
||||
*kv_cache_strides,
|
||||
num_heads=num_heads,
|
||||
content_size=content_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
|
||||
def gather_kv_caches(
|
||||
kv_caches_ptrs: torch.Tensor,
|
||||
total_token_in_kvcache: int,
|
||||
dst_tensor: torch.Tensor,
|
||||
token_indices: list[int],
|
||||
tokens_per_block: int,
|
||||
num_heads: int,
|
||||
content_size: int,
|
||||
kv_cache_strides: tuple[int, ...],
|
||||
) -> None:
|
||||
"""Gather KV cache data from KV cache storage to destination tensor.
|
||||
|
||||
Args:
|
||||
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
|
||||
total_token_in_kvcache: Total number of tokens in KV cache
|
||||
dst_tensor: Destination ``[L, H, N, C]`` tensor
|
||||
token_indices: List of token positions to gather
|
||||
tokens_per_block: Number of stored states in each cache block
|
||||
num_heads: Size of the H axis
|
||||
content_size: Size of the C axis
|
||||
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
|
||||
|
||||
"""
|
||||
num_layers = kv_caches_ptrs.shape[0]
|
||||
num_tokens_in_block = len(token_indices)
|
||||
|
||||
assert dst_tensor.shape == (
|
||||
num_layers,
|
||||
num_heads,
|
||||
num_tokens_in_block,
|
||||
content_size,
|
||||
)
|
||||
assert len(kv_cache_strides) == 4
|
||||
|
||||
device = dst_tensor.device
|
||||
token_indices_tensor = async_tensor_h2d(
|
||||
token_indices, device=device, dtype=torch.int32
|
||||
)
|
||||
|
||||
grid = (num_layers, num_tokens_in_block)
|
||||
BLOCK_SIZE = 128
|
||||
|
||||
kv_cache_gather_kernel[grid](
|
||||
kv_caches_ptrs,
|
||||
dst_tensor,
|
||||
token_indices_tensor,
|
||||
num_tokens_in_block,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
tokens_per_block,
|
||||
*kv_cache_strides,
|
||||
num_heads=num_heads,
|
||||
content_size=content_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
|
||||
class CopyBufferAllocator:
|
||||
"""Memory pool for tensor buffers to avoid frequent allocation/deallocation."""
|
||||
|
||||
def __init__(
|
||||
self, device: torch.device, dtype: torch.dtype, shape: list, max_count: int
|
||||
):
|
||||
self._shape = shape
|
||||
self._max_count = max_count
|
||||
self._device = device
|
||||
self._free_buffers = [
|
||||
torch.empty(shape, dtype=dtype, device=device) for _ in range(max_count)
|
||||
]
|
||||
self._inuse_count = 0
|
||||
|
||||
def alloc_buffer(self, count: int) -> list[torch.Tensor] | None:
|
||||
"""Allocate buffers from the pool."""
|
||||
if count == 0:
|
||||
return []
|
||||
|
||||
if self._inuse_count + count <= self._max_count:
|
||||
self._inuse_count += count
|
||||
result = self._free_buffers[-count:]
|
||||
del self._free_buffers[-count:]
|
||||
return result
|
||||
return None
|
||||
|
||||
def free_buffer(self, buffers: list[torch.Tensor]) -> None:
|
||||
"""Return buffers to the pool."""
|
||||
if not buffers:
|
||||
return
|
||||
|
||||
if self._inuse_count >= len(buffers):
|
||||
self._inuse_count -= len(buffers)
|
||||
self._free_buffers.extend(buffers)
|
||||
else:
|
||||
raise RuntimeError("Attempted to free more buffers than allocated")
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -1,133 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
HF3FS_AVAILABLE = True
|
||||
|
||||
|
||||
class Hf3fsClient:
|
||||
"""Mock HF3FS client using file backend for debugging and testing."""
|
||||
|
||||
def __init__(self, path: str, size: int, bytes_per_page: int, entries: int):
|
||||
self._size = size
|
||||
self._bytes_per_page = bytes_per_page
|
||||
self._entries = entries
|
||||
self._file_path = path
|
||||
|
||||
self._ensure_file_exists()
|
||||
logger.debug("Initialized mock HF3FS client: %s (%d bytes)", path, size)
|
||||
|
||||
def _ensure_file_exists(self) -> None:
|
||||
"""Create file if it doesn't exist."""
|
||||
if not os.path.exists(self._file_path):
|
||||
with open(self._file_path, "w+b") as f:
|
||||
f.truncate(self._size)
|
||||
|
||||
def batch_read(self, offsets: list[int], tensors: list[torch.Tensor]) -> list[int]:
|
||||
"""Read data from file at specified offsets into tensors."""
|
||||
results = []
|
||||
|
||||
try:
|
||||
with open(self._file_path, "rb") as f:
|
||||
for offset, tensor in zip(offsets, tensors):
|
||||
num_bytes = tensor.numel() * tensor.element_size()
|
||||
|
||||
if offset < 0 or offset + num_bytes > self._size:
|
||||
results.append(-1)
|
||||
continue
|
||||
|
||||
f.seek(offset)
|
||||
buffer_data = f.read(num_bytes)
|
||||
|
||||
if len(buffer_data) == num_bytes == self._bytes_per_page:
|
||||
tensor_data = self._convert_buffer_to_tensor(
|
||||
buffer_data, tensor.dtype
|
||||
)
|
||||
tensor.copy_(
|
||||
tensor_data.reshape(tensor.shape).to(tensor.device)
|
||||
)
|
||||
results.append(self._bytes_per_page)
|
||||
else:
|
||||
logger.error(
|
||||
"Read size mismatch: got %d, expected %d",
|
||||
len(buffer_data),
|
||||
num_bytes,
|
||||
)
|
||||
results.append(-1)
|
||||
except Exception as e:
|
||||
logger.error("Batch read error: %s", e)
|
||||
results.extend([-1] * (len(offsets) - len(results)))
|
||||
|
||||
return results
|
||||
|
||||
def _convert_buffer_to_tensor(
|
||||
self, buffer_data: bytes, dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""Convert buffer data to tensor with proper dtype handling."""
|
||||
if dtype == torch.bfloat16:
|
||||
tensor_data = torch.frombuffer(buffer_data, dtype=torch.uint16)
|
||||
return tensor_data.view(dtype=torch.bfloat16)
|
||||
else:
|
||||
return torch.frombuffer(buffer_data, dtype=dtype)
|
||||
|
||||
def batch_write(
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
|
||||
) -> list[int]:
|
||||
"""Write data from tensors to file at specified offsets."""
|
||||
results = []
|
||||
|
||||
try:
|
||||
torch.cuda.current_stream().wait_event(event)
|
||||
|
||||
# Convert tensors to bytes
|
||||
data_bytes_list = [self._tensor_to_bytes(tensor) for tensor in tensors]
|
||||
|
||||
# Write to file
|
||||
with open(self._file_path, "r+b") as f:
|
||||
for offset, data_bytes in zip(offsets, data_bytes_list):
|
||||
if offset < 0 or offset + len(data_bytes) > self._size:
|
||||
results.append(-1)
|
||||
continue
|
||||
|
||||
f.seek(offset)
|
||||
bytes_written = f.write(data_bytes)
|
||||
|
||||
if bytes_written == len(data_bytes) == self._bytes_per_page:
|
||||
results.append(self._bytes_per_page)
|
||||
else:
|
||||
logger.error(
|
||||
"Write size mismatch: wrote %d, expected %d",
|
||||
bytes_written,
|
||||
self._bytes_per_page,
|
||||
)
|
||||
results.append(-1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Batch write error: %s", e)
|
||||
results.extend([-1] * (len(offsets) - len(results)))
|
||||
|
||||
return results
|
||||
|
||||
def _tensor_to_bytes(self, tensor: torch.Tensor) -> bytes:
|
||||
"""Convert tensor to bytes with proper dtype handling."""
|
||||
cpu_tensor = tensor.cpu()
|
||||
if cpu_tensor.dtype == torch.bfloat16:
|
||||
return cpu_tensor.view(dtype=torch.uint16).numpy().tobytes()
|
||||
else:
|
||||
return cpu_tensor.numpy().tobytes()
|
||||
|
||||
def get_size(self) -> int:
|
||||
"""Get the total size of the storage file."""
|
||||
return self._size
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the client (no-op for file backend)."""
|
||||
pass
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Flush any pending writes (no-op for file backend)."""
|
||||
pass
|
||||
@@ -1,57 +0,0 @@
|
||||
#include <cuda_runtime.h>
|
||||
#include <torch/extension.h>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
void read_shm(const torch::Tensor& shm, const torch::Tensor& pin,
|
||||
std::vector<torch::Tensor> dst, uint64_t stream_ptr) {
|
||||
py::gil_scoped_release release;
|
||||
|
||||
cudaStream_t stream = reinterpret_cast<cudaStream_t>(stream_ptr);
|
||||
|
||||
// Copy from shared memory to pinned memory
|
||||
char* shm_ptr = static_cast<char*>(shm.data_ptr());
|
||||
char* src_ptr = static_cast<char*>(pin.data_ptr());
|
||||
std::memcpy(src_ptr, shm_ptr, shm.numel() * shm.element_size());
|
||||
|
||||
// Copy from pinned memory to GPU tensors
|
||||
size_t current = 0;
|
||||
for (size_t i = 0; i < dst.size(); ++i) {
|
||||
auto& t = dst[i];
|
||||
size_t t_bytes = t.numel() * t.element_size();
|
||||
char* dst_ptr = static_cast<char*>(t.data_ptr());
|
||||
cudaMemcpyAsync(dst_ptr, src_ptr + current, t_bytes, cudaMemcpyHostToDevice,
|
||||
stream);
|
||||
current += t_bytes;
|
||||
}
|
||||
cudaStreamSynchronize(stream);
|
||||
}
|
||||
|
||||
void write_shm(const std::vector<torch::Tensor> src, torch::Tensor& shm,
|
||||
const torch::Tensor& pin, uint64_t stream_ptr) {
|
||||
py::gil_scoped_release release;
|
||||
|
||||
cudaStream_t stream = reinterpret_cast<cudaStream_t>(stream_ptr);
|
||||
|
||||
// Copy from GPU tensors to pinned memory
|
||||
char* dst_ptr = static_cast<char*>(pin.data_ptr());
|
||||
size_t current = 0;
|
||||
for (size_t i = 0; i < src.size(); ++i) {
|
||||
auto& t = src[i];
|
||||
size_t t_bytes = t.numel() * t.element_size();
|
||||
char* src_ptr = static_cast<char*>(t.data_ptr());
|
||||
cudaMemcpyAsync(dst_ptr + current, src_ptr, t_bytes, cudaMemcpyDeviceToHost,
|
||||
stream);
|
||||
current += t_bytes;
|
||||
}
|
||||
cudaStreamSynchronize(stream);
|
||||
|
||||
// Copy from pinned memory to shared memory
|
||||
char* shm_ptr = static_cast<char*>(shm.data_ptr());
|
||||
std::memcpy(shm_ptr, dst_ptr, shm.numel() * shm.element_size());
|
||||
}
|
||||
|
||||
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
|
||||
m.def("read_shm", &read_shm, "Read tensors from shared memory");
|
||||
m.def("write_shm", &write_shm, "Write tensors to shared memory");
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,408 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Connector boundary for HiSparse scheduler and worker state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
HiSparsePromMetrics,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
|
||||
HiSparseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.hisparse.layout import get_hisparse_steady_state_concurrency
|
||||
from vllm.v1.hisparse.types import (
|
||||
SparseKVOffloadCommand,
|
||||
SparseKVResidencyUpdate,
|
||||
SparseKVRowMirror,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.hisparse.coordinator import HiSparseCoordinator
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseConnectorMetadata(KVConnectorMetadata):
|
||||
command: SparseKVOffloadCommand | None
|
||||
host_block_copies: tuple[KVCacheBlockCopy, ...]
|
||||
source_block_ids: tuple[int, ...]
|
||||
row_mirrors: dict[str, tuple[SparseKVRowMirror, ...]]
|
||||
all_context_pages_resident: bool
|
||||
residency_updates: dict[str, SparseKVResidencyUpdate]
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseConnectorWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
enqueued_transfer_counts: dict[int, int]
|
||||
completed_transfer_counts: dict[int, int]
|
||||
completed_host_copy_dst_ids: tuple[int, ...] = ()
|
||||
|
||||
def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetadata:
|
||||
assert isinstance(other, HiSparseConnectorWorkerMetadata)
|
||||
|
||||
def add_counts(first: dict[int, int], second: dict[int, int]) -> dict[int, int]:
|
||||
combined = first.copy()
|
||||
for transfer_id, count in second.items():
|
||||
combined[transfer_id] = combined.get(transfer_id, 0) + count
|
||||
return combined
|
||||
|
||||
return HiSparseConnectorWorkerMetadata(
|
||||
enqueued_transfer_counts=add_counts(
|
||||
self.enqueued_transfer_counts, other.enqueued_transfer_counts
|
||||
),
|
||||
completed_transfer_counts=add_counts(
|
||||
self.completed_transfer_counts, other.completed_transfer_counts
|
||||
),
|
||||
# Every worker runs the same copies, so a union is the ack.
|
||||
completed_host_copy_dst_ids=tuple(
|
||||
{*self.completed_host_copy_dst_ids, *other.completed_host_copy_dst_ids}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class HiSparseConnectorScheduler:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
async_speculative: bool,
|
||||
draft_kv_lookahead: int = 0,
|
||||
) -> None:
|
||||
self.coordinator: HiSparseCoordinator | None = None
|
||||
self.async_speculative = async_speculative
|
||||
self.draft_kv_lookahead = draft_kv_lookahead
|
||||
self.requests: dict[str, Request] = {}
|
||||
|
||||
def bind_coordinator(self, coordinator: HiSparseCoordinator) -> None:
|
||||
assert self.coordinator is None
|
||||
self.coordinator = coordinator
|
||||
|
||||
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
|
||||
if self.coordinator is None:
|
||||
return None
|
||||
host_pool = self.coordinator.get_host_block_pool()
|
||||
assert host_pool is not None
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_host_usage(
|
||||
host_pool.get_usage(), len(self.coordinator.pending_spills)
|
||||
)
|
||||
if host_pool.metrics_collector is not None:
|
||||
stats.record_host_evictions(host_pool.metrics_collector.drain_events())
|
||||
return stats
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> HiSparseConnectorMetadata:
|
||||
assert self.coordinator is not None
|
||||
# HiSparse rebinds worker state every step, so the load must start
|
||||
# before the forward rather than being deferred to post-forward.
|
||||
scheduler_output.has_sync_kv_loads = True
|
||||
num_computed_tokens = {
|
||||
request.req_id: request.num_computed_tokens
|
||||
for request in scheduler_output.scheduled_new_reqs
|
||||
}
|
||||
num_computed_tokens.update(
|
||||
zip(
|
||||
scheduler_output.scheduled_cached_reqs.req_ids,
|
||||
scheduler_output.scheduled_cached_reqs.num_computed_tokens,
|
||||
)
|
||||
)
|
||||
scheduled_requests = tuple(
|
||||
(
|
||||
request_id,
|
||||
num_computed_tokens[request_id],
|
||||
scheduled_count,
|
||||
)
|
||||
for request_id, scheduled_count in (
|
||||
scheduler_output.num_scheduled_tokens.items()
|
||||
)
|
||||
)
|
||||
# Runs first: it plans this step's transfers and residency changes.
|
||||
self.coordinator.advance_scheduled(
|
||||
(
|
||||
request_id,
|
||||
min(start + count, self.requests[request_id].num_tokens),
|
||||
)
|
||||
for request_id, start, count in scheduled_requests
|
||||
)
|
||||
command = self.coordinator.build_offload_command()
|
||||
host_block_copies = self.coordinator.take_host_block_copies()
|
||||
source_group_id = self.coordinator.host_group_id
|
||||
assert source_group_id is not None
|
||||
source_block_ids = [
|
||||
block_id
|
||||
for request in scheduler_output.scheduled_new_reqs
|
||||
for block_id in request.block_ids[source_group_id]
|
||||
]
|
||||
for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids:
|
||||
if new_block_ids is not None:
|
||||
source_block_ids.extend(new_block_ids[source_group_id])
|
||||
row_mirrors = {}
|
||||
for request_id, scheduled_start, scheduled_count in scheduled_requests:
|
||||
mirror_start = scheduled_start
|
||||
if self.async_speculative:
|
||||
mirror_start = max(
|
||||
0,
|
||||
scheduled_start - self.requests[request_id].num_output_placeholders,
|
||||
)
|
||||
row_mirrors[request_id] = self.coordinator.build_row_mirrors(
|
||||
(
|
||||
(
|
||||
request_id,
|
||||
mirror_start,
|
||||
scheduled_count
|
||||
+ scheduled_start
|
||||
- mirror_start
|
||||
+ self.draft_kv_lookahead,
|
||||
),
|
||||
),
|
||||
)
|
||||
return HiSparseConnectorMetadata(
|
||||
command,
|
||||
host_block_copies,
|
||||
tuple(source_block_ids),
|
||||
row_mirrors,
|
||||
self.coordinator.all_context_pages_resident(scheduled_requests),
|
||||
self.coordinator.take_residency_updates(
|
||||
scheduler_output.num_scheduled_tokens
|
||||
),
|
||||
)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
assert self.coordinator is not None
|
||||
for request_id in connector_output.finished_recving or ():
|
||||
self.coordinator.finish_host_import(
|
||||
request_id, failed=request_id in connector_output.failed_recving
|
||||
)
|
||||
metadata = connector_output.kv_connector_worker_meta
|
||||
if metadata is None:
|
||||
return
|
||||
assert isinstance(metadata, HiSparseConnectorWorkerMetadata)
|
||||
self.coordinator.release_completed_host_copies(
|
||||
metadata.completed_host_copy_dst_ids
|
||||
)
|
||||
self.coordinator.update_spills(
|
||||
metadata.enqueued_transfer_counts,
|
||||
metadata.completed_transfer_counts,
|
||||
)
|
||||
|
||||
|
||||
class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""Join the scheduler coordinator to the worker's transfer engine."""
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str:
|
||||
return "BLHNC"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
if kv_cache_config.hisparse_host_num_blocks is None:
|
||||
raise ValueError("HiSparseConnector requires a HiSparse host pool")
|
||||
self.connector_scheduler: HiSparseConnectorScheduler | None = None
|
||||
self.connector_worker: HiSparseConnectorWorker | None = None
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
self.connector_scheduler = HiSparseConnectorScheduler(
|
||||
async_speculative=bool(
|
||||
vllm_config.scheduler_config.async_scheduling
|
||||
and speculative_config is not None
|
||||
),
|
||||
draft_kv_lookahead=vllm_config.num_lookahead_tokens,
|
||||
)
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
steady_concurrency = get_hisparse_steady_state_concurrency(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
logger.info_once(
|
||||
"HiSparse steady-state maximum concurrency for %s tokens per "
|
||||
"request: %.2fx (running requests reading from host).",
|
||||
f"{max_model_len:,}",
|
||||
steady_concurrency,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_worker = HiSparseConnectorWorker(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported KV connector role: {role}")
|
||||
|
||||
def bind_kv_cache_manager(self, kv_cache_manager: KVCacheManager) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.bind_coordinator(
|
||||
get_hisparse_coordinator(kv_cache_manager)
|
||||
)
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
return False
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
assert self.connector_scheduler.coordinator is not None
|
||||
return self.connector_scheduler.coordinator.has_pending_work()
|
||||
|
||||
def has_pending_block_frees(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
assert self.connector_scheduler.coordinator is not None
|
||||
return self.connector_scheduler.coordinator.has_pending_reclamation()
|
||||
|
||||
def finish_forward(self) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.finish_forward()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def reset_capture_state(self) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.reset_hot_state()
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is not None:
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
if self.connector_scheduler is not None:
|
||||
return self.connector_scheduler.get_kv_connector_stats()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
HiSparseKVConnectorStats(data=data)
|
||||
if data is not None
|
||||
else HiSparseKVConnectorStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return HiSparsePromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, HiSparseConnectorMetadata)
|
||||
request_state_indices = kwargs.get("request_state_indices")
|
||||
assert request_state_indices is None or isinstance(
|
||||
request_state_indices, torch.Tensor
|
||||
)
|
||||
request_ids = kwargs.get("request_ids")
|
||||
assert request_ids is None or isinstance(request_ids, list)
|
||||
attn_metadata = kwargs.get("attn_metadata")
|
||||
assert attn_metadata is None or isinstance(attn_metadata, dict)
|
||||
self.connector_worker.start_step(
|
||||
metadata,
|
||||
request_state_indices,
|
||||
request_ids,
|
||||
num_tokens=int(kwargs.get("num_tokens") or 0),
|
||||
)
|
||||
self.connector_worker.prepare_forward(attn_metadata)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
return
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
return
|
||||
|
||||
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
|
||||
assert self.connector_worker is not None
|
||||
enqueued, completed = self.connector_worker.take_transfer_updates()
|
||||
host_copies = self.connector_worker.take_completed_host_copies()
|
||||
if not enqueued and not completed and not host_copies:
|
||||
return None
|
||||
return HiSparseConnectorWorkerMetadata(
|
||||
enqueued_transfer_counts={transfer_id: 1 for transfer_id in enqueued},
|
||||
completed_transfer_counts={transfer_id: 1 for transfer_id in completed},
|
||||
completed_host_copy_dst_ids=tuple(host_copies),
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.shutdown()
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.requests[request.request_id] = request
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.requests.pop(request.request_id, None)
|
||||
return False, None
|
||||
@@ -1,214 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stats and Prometheus metrics for the HiSparse connector."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.v1.metrics.buckets import histogram_buckets
|
||||
from vllm.v1.metrics.stats import KVCacheEvictionEvent
|
||||
from vllm.v1.metrics.utils import create_metric_per_engine
|
||||
|
||||
_HISPARSE_LEVEL_KEYS = frozenset({"host_cache_usage_perc", "pending_page_transfers"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseKVConnectorStats(KVConnectorStats):
|
||||
"""Container for HiSparse hot-buffer residency metrics.
|
||||
|
||||
Each list entry is the delta recorded over one device counter snapshot
|
||||
interval, so a list can hold multiple snapshots per logging interval.
|
||||
The host-tier gauges are level values, not deltas: aggregation keeps
|
||||
only the most recent observation.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
# Empty container init, no data is passed in.
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
# Must be serializable
|
||||
self.data: dict[str, list[int | float]] = {
|
||||
"cache_hits": [],
|
||||
"cache_misses": [],
|
||||
"host_to_device_bytes": [],
|
||||
"host_cache_usage_perc": [],
|
||||
"pending_page_transfers": [],
|
||||
"host_block_lifetime_seconds": [],
|
||||
"host_block_idle_before_evict_seconds": [],
|
||||
"host_block_reuse_gap_seconds": [],
|
||||
}
|
||||
|
||||
def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int):
|
||||
self.data["cache_hits"].append(hits)
|
||||
self.data["cache_misses"].append(misses)
|
||||
self.data["host_to_device_bytes"].append(host_to_device_bytes)
|
||||
|
||||
def record_host_usage(self, usage: float, pending_page_transfers: int) -> None:
|
||||
self.data["host_cache_usage_perc"].append(usage)
|
||||
self.data["pending_page_transfers"].append(pending_page_transfers)
|
||||
|
||||
def record_host_evictions(self, events: list[KVCacheEvictionEvent]) -> None:
|
||||
for event in events:
|
||||
self.data["host_block_lifetime_seconds"].append(event.lifetime_seconds)
|
||||
self.data["host_block_idle_before_evict_seconds"].append(event.idle_seconds)
|
||||
self.data["host_block_reuse_gap_seconds"].extend(event.reuse_gaps_seconds)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
if k in _HISPARSE_LEVEL_KEYS:
|
||||
accumulator.extend(v[-1:])
|
||||
else:
|
||||
accumulator.extend(v)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
# Compute compact representative stats suitable for CLI logging.
|
||||
reduced: dict[str, int | float] = {
|
||||
"HiSparse hot-buffer hits": sum(self.data["cache_hits"]),
|
||||
"HiSparse hot-buffer misses": sum(self.data["cache_misses"]),
|
||||
"HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]),
|
||||
}
|
||||
if self.data["host_cache_usage_perc"]:
|
||||
usage = self.data["host_cache_usage_perc"][-1]
|
||||
reduced["HiSparse host KV cache usage %"] = round(100 * usage, 1)
|
||||
return reduced
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return all(len(values) == 0 for values in self.data.values())
|
||||
|
||||
|
||||
class HiSparsePromMetrics(KVConnectorPromMetrics):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
|
||||
counter_cache_hits = self._counter_cls(
|
||||
name="vllm:hisparse_cache_hits",
|
||||
documentation="Number of HiSparse device hot-buffer hits.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
counter_cache_misses = self._counter_cls(
|
||||
name="vllm:hisparse_cache_misses",
|
||||
documentation="Number of HiSparse device hot-buffer misses.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
counter_host_to_device_bytes = self._counter_cls(
|
||||
name="vllm:hisparse_host_to_device_bytes",
|
||||
documentation=(
|
||||
"Bytes transferred from host KV storage to HiSparse hot buffers."
|
||||
),
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_counters: dict[str, Any] = {
|
||||
"cache_hits": create_metric_per_engine(
|
||||
counter_cache_hits, self.per_engine_labelvalues
|
||||
),
|
||||
"cache_misses": create_metric_per_engine(
|
||||
counter_cache_misses, self.per_engine_labelvalues
|
||||
),
|
||||
"host_to_device_bytes": create_metric_per_engine(
|
||||
counter_host_to_device_bytes, self.per_engine_labelvalues
|
||||
),
|
||||
}
|
||||
|
||||
gauge_host_cache_usage = self._gauge_cls(
|
||||
name="vllm:hisparse_host_cache_usage_perc",
|
||||
documentation=(
|
||||
"HiSparse host KV-cache usage. 1 means 100 percent usage. "
|
||||
"Evictable cached blocks count as free."
|
||||
),
|
||||
multiprocess_mode="mostrecent",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
gauge_pending_page_transfers = self._gauge_cls(
|
||||
name="vllm:hisparse_pending_page_transfers",
|
||||
documentation=(
|
||||
"HiSparse page transfers (write-backs and restores) not yet completed."
|
||||
),
|
||||
multiprocess_mode="mostrecent",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_gauges: dict[str, Any] = {
|
||||
"host_cache_usage_perc": create_metric_per_engine(
|
||||
gauge_host_cache_usage, self.per_engine_labelvalues
|
||||
),
|
||||
"pending_page_transfers": create_metric_per_engine(
|
||||
gauge_pending_page_transfers, self.per_engine_labelvalues
|
||||
),
|
||||
}
|
||||
|
||||
self.hisparse_histograms: dict[str, Any] = {}
|
||||
if vllm_config.observability_config.kv_cache_metrics:
|
||||
residency_buckets = histogram_buckets(
|
||||
"kv_cache_residency",
|
||||
overrides=vllm_config.observability_config.custom_histogram_buckets,
|
||||
)
|
||||
histogram_host_block_lifetime = self._histogram_cls(
|
||||
name="vllm:hisparse_host_block_lifetime_seconds",
|
||||
documentation=(
|
||||
"Histogram of HiSparse host KV block lifetime from allocation "
|
||||
"to eviction. Sampled metrics (controlled by "
|
||||
"--kv-cache-metrics-sample)."
|
||||
),
|
||||
buckets=residency_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
histogram_host_block_idle_before_evict = self._histogram_cls(
|
||||
name="vllm:hisparse_host_block_idle_before_evict_seconds",
|
||||
documentation=(
|
||||
"Histogram of HiSparse host KV block idle time before eviction. "
|
||||
"Sampled metrics (controlled by --kv-cache-metrics-sample)."
|
||||
),
|
||||
buckets=residency_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
histogram_host_block_reuse_gap = self._histogram_cls(
|
||||
name="vllm:hisparse_host_block_reuse_gap_seconds",
|
||||
documentation=(
|
||||
"Histogram of time gaps between consecutive HiSparse host KV "
|
||||
"block accesses. Sampled metrics (controlled by "
|
||||
"--kv-cache-metrics-sample)."
|
||||
),
|
||||
buckets=residency_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.hisparse_histograms = {
|
||||
"host_block_lifetime_seconds": create_metric_per_engine(
|
||||
histogram_host_block_lifetime, self.per_engine_labelvalues
|
||||
),
|
||||
"host_block_idle_before_evict_seconds": create_metric_per_engine(
|
||||
histogram_host_block_idle_before_evict, self.per_engine_labelvalues
|
||||
),
|
||||
"host_block_reuse_gap_seconds": create_metric_per_engine(
|
||||
histogram_host_block_reuse_gap, self.per_engine_labelvalues
|
||||
),
|
||||
}
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
for name, counter in self.hisparse_counters.items():
|
||||
for value in transfer_stats_data.get(name, []):
|
||||
counter[engine_idx].inc(value)
|
||||
for name, gauge in self.hisparse_gauges.items():
|
||||
values = transfer_stats_data.get(name, [])
|
||||
if values:
|
||||
gauge[engine_idx].set(values[-1])
|
||||
for name, histogram in self.hisparse_histograms.items():
|
||||
for value in transfer_stats_data.get(name, []):
|
||||
histogram[engine_idx].observe(value)
|
||||
@@ -1,978 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HiSparse worker-side host/hot state and data movement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparseCacheHandle,
|
||||
drop_hisparse_host_pool,
|
||||
release_pinned_state,
|
||||
update_hisparse_residency,
|
||||
)
|
||||
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
HiSparseHotSpec,
|
||||
KVCacheConfig,
|
||||
)
|
||||
from vllm.v1.worker.utils import copy_kv_cache_blocks_inplace
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
|
||||
|
||||
class _DMADescriptors(NamedTuple):
|
||||
src: torch.Tensor
|
||||
dst: torch.Tensor
|
||||
sizes: torch.Tensor
|
||||
src_np: np.ndarray
|
||||
dst_np: np.ndarray
|
||||
sizes_np: np.ndarray
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SlotMappingStaging:
|
||||
stream: torch.Stream
|
||||
event: torch.Event
|
||||
slots: torch.Tensor
|
||||
candidates: tuple[SparseKVRowMirror, ...] = ()
|
||||
num_tokens: int = 0
|
||||
source_index: int = 0
|
||||
|
||||
|
||||
def _allocate_dma_descriptors(size: int) -> _DMADescriptors:
|
||||
src, dst, sizes = (torch.empty(size, dtype=torch.int64) for _ in range(3))
|
||||
return _DMADescriptors(
|
||||
src,
|
||||
dst,
|
||||
sizes,
|
||||
src.numpy(),
|
||||
dst.numpy(),
|
||||
sizes.numpy(),
|
||||
)
|
||||
|
||||
|
||||
_METRICS_INTERVAL = 2000
|
||||
|
||||
|
||||
def _get_hisparse_cache(
|
||||
forward_context: dict[str, Any], layer_name: str
|
||||
) -> HiSparseCacheHandle:
|
||||
attention_layer = forward_context[layer_name]
|
||||
hisparse_cache = attention_layer.hisparse_cache
|
||||
assert hisparse_cache is not None
|
||||
return hisparse_cache
|
||||
|
||||
|
||||
def _flatten_row_mirrors(
|
||||
row_mirrors: Mapping[str, tuple[SparseKVRowMirror, ...]],
|
||||
request_ids: Sequence[str] | None,
|
||||
) -> tuple[SparseKVRowMirror, ...]:
|
||||
ordered_ids = row_mirrors if request_ids is None else request_ids
|
||||
return tuple(
|
||||
mirror
|
||||
for request_id in ordered_ids
|
||||
for mirror in row_mirrors.get(request_id, ())
|
||||
)
|
||||
|
||||
|
||||
def _select_written_row_mirrors(
|
||||
candidates: tuple[SparseKVRowMirror, ...],
|
||||
source_slots: np.ndarray,
|
||||
source_index: int,
|
||||
) -> tuple[SparseKVRowMirror, ...]:
|
||||
"""Select and coalesce GPU-written rows from a scheduler-owned envelope."""
|
||||
source_slots = source_slots[source_slots >= 0]
|
||||
if source_slots.size == 0 or not candidates:
|
||||
return ()
|
||||
starts = np.asarray([mirror.source_starts for mirror in candidates], dtype=np.int64)
|
||||
counts = np.fromiter((mirror.num_rows for mirror in candidates), dtype=np.int64)
|
||||
destinations = np.fromiter(
|
||||
[mirror.destination_start for mirror in candidates], dtype=np.int64
|
||||
)
|
||||
if source_index >= starts.shape[1]:
|
||||
raise RuntimeError("HiSparse row-mirror source index is out of range.")
|
||||
source_starts = starts[:, source_index]
|
||||
order = np.argsort(source_starts)
|
||||
matches = np.searchsorted(source_starts[order], source_slots, side="right") - 1
|
||||
candidate_ids = order[np.maximum(matches, 0)]
|
||||
offsets = source_slots - source_starts[candidate_ids]
|
||||
valid = (matches >= 0) & (offsets < counts[candidate_ids])
|
||||
if not np.all(valid):
|
||||
# A page the scheduler could not map has no host destination, so its
|
||||
# rows are simply not mirrorable this step.
|
||||
source_slots = source_slots[valid]
|
||||
candidate_ids = candidate_ids[valid]
|
||||
offsets = offsets[valid]
|
||||
if source_slots.size == 0:
|
||||
return ()
|
||||
sources = starts[candidate_ids] + offsets[:, None]
|
||||
destinations = destinations[candidate_ids] + offsets
|
||||
boundaries = (
|
||||
np.flatnonzero(
|
||||
(candidate_ids[1:] != candidate_ids[:-1])
|
||||
| (np.diff(destinations) != 1)
|
||||
| np.any(np.diff(sources, axis=0) != 1, axis=1)
|
||||
)
|
||||
+ 1
|
||||
)
|
||||
run_starts = np.concatenate(([0], boundaries))
|
||||
run_ends = np.concatenate((boundaries, [destinations.size]))
|
||||
return tuple(
|
||||
SparseKVRowMirror(
|
||||
source_starts=tuple(int(value) for value in sources[start]),
|
||||
destination_start=int(destinations[start]),
|
||||
num_rows=int(end - start),
|
||||
)
|
||||
for start, end in zip(run_starts, run_ends, strict=True)
|
||||
)
|
||||
|
||||
|
||||
def _is_hisparse_host_writer(
|
||||
shared_host_region: SharedOffloadRegion | None,
|
||||
) -> bool:
|
||||
return shared_host_region is None or get_tensor_model_parallel_rank() == 0
|
||||
|
||||
|
||||
def _create_hisparse_host_events(
|
||||
shared_host_region: SharedOffloadRegion | None,
|
||||
is_host_writer: bool,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Event, torch.Event]:
|
||||
if shared_host_region is None:
|
||||
return torch.Event(), torch.Event()
|
||||
|
||||
events: tuple[torch.Event, torch.Event] | None = None
|
||||
ipc_handles = None
|
||||
if is_host_writer:
|
||||
events = (
|
||||
torch.cuda.Event(interprocess=True),
|
||||
torch.cuda.Event(interprocess=True),
|
||||
)
|
||||
stream = current_stream()
|
||||
for event in events:
|
||||
event.record(stream)
|
||||
ipc_handles = tuple(event.ipc_handle() for event in events)
|
||||
ipc_handles = get_tp_group().broadcast_object(ipc_handles, src=0)
|
||||
if events is None:
|
||||
events = (
|
||||
torch.cuda.Event.from_ipc_handle(device, ipc_handles[0]),
|
||||
torch.cuda.Event.from_ipc_handle(device, ipc_handles[1]),
|
||||
)
|
||||
return events
|
||||
|
||||
|
||||
class HiSparseConnectorWorker:
|
||||
"""Own HiSparse host/hot state and execute its worker-side transfers."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._initialized = False
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
forward_context = self.vllm_config.compilation_config.static_forward_context
|
||||
cache_handles: list[HiSparseCacheHandle] = []
|
||||
cache_layer_names: list[str] = []
|
||||
for group in self.kv_cache_config.kv_cache_groups:
|
||||
if not isinstance(group.kv_cache_spec, HiSparseHotSpec):
|
||||
continue
|
||||
for cache_name in group.layer_names:
|
||||
assert cache_name.endswith(HISPARSE_HOT_SUFFIX)
|
||||
layer_name = cache_name[: -len(HISPARSE_HOT_SUFFIX)]
|
||||
cache_handles.append(_get_hisparse_cache(forward_context, layer_name))
|
||||
cache_layer_names.append(layer_name)
|
||||
|
||||
if not cache_handles:
|
||||
raise RuntimeError("HiSparse connector found no hot-cache handles.")
|
||||
hot_backings: dict[int, torch.Tensor] = {}
|
||||
registered_host_pools: dict[int, torch.Tensor] = {}
|
||||
shared_host_regions: dict[int, SharedOffloadRegion] = {}
|
||||
for cache in cache_handles:
|
||||
hot_backing = cache.runtime.hot_backing
|
||||
hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing
|
||||
registered_pool = cache.runtime.registered_host_pool
|
||||
registered_host_pools[registered_pool.data_ptr()] = registered_pool
|
||||
if (region := cache.runtime.shared_host_region) is not None:
|
||||
shared_host_regions[id(region)] = region
|
||||
if len(hot_backings) != 1:
|
||||
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
|
||||
hot_backing = next(iter(hot_backings.values()))
|
||||
if len(shared_host_regions) > 1:
|
||||
raise RuntimeError("HiSparse caches must share one host region.")
|
||||
shared_host_region = next(iter(shared_host_regions.values()), None)
|
||||
pinned_host_pools = (
|
||||
[]
|
||||
if shared_host_region is not None
|
||||
else list(registered_host_pools.values())
|
||||
)
|
||||
is_host_writer = _is_hisparse_host_writer(shared_host_region)
|
||||
|
||||
resident = cache_handles[0].view
|
||||
assert resident is not None
|
||||
host_num_blocks = self.kv_cache_config.hisparse_host_num_blocks
|
||||
assert host_num_blocks is not None
|
||||
try:
|
||||
self.initialize(
|
||||
cache_handles,
|
||||
cache_layer_names,
|
||||
hot_backing,
|
||||
self.vllm_config.scheduler_config.max_num_seqs,
|
||||
host_num_blocks,
|
||||
hot_backing.device,
|
||||
pinned_host_pools,
|
||||
shared_host_region=shared_host_region,
|
||||
is_host_writer=is_host_writer,
|
||||
)
|
||||
except Exception:
|
||||
release_pinned_state(
|
||||
[cache.runtime for cache in cache_handles],
|
||||
pinned_host_pools,
|
||||
shared_host_region,
|
||||
)
|
||||
drop_hisparse_host_pool(self.vllm_config)
|
||||
raise
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
cache_handles: list[HiSparseCacheHandle],
|
||||
cache_layer_names: list[str],
|
||||
hot_backing: torch.Tensor,
|
||||
max_num_reqs: int,
|
||||
host_num_blocks: int,
|
||||
device: torch.device,
|
||||
pinned_host_pools: list[torch.Tensor],
|
||||
*,
|
||||
shared_host_region: SharedOffloadRegion | None = None,
|
||||
is_host_writer: bool = True,
|
||||
) -> None:
|
||||
if self._initialized:
|
||||
raise RuntimeError("HiSparse connector worker is already initialized.")
|
||||
resident = cache_handles[0].view
|
||||
assert resident is not None
|
||||
self.kernel_block_size = resident.block_size
|
||||
self.host_num_blocks = host_num_blocks
|
||||
self.pinned_host_pools = pinned_host_pools
|
||||
self.shared_host_region = shared_host_region
|
||||
self.is_host_writer = is_host_writer
|
||||
self.dma_stream = (
|
||||
torch.cuda.Stream(device=device) if self.is_host_writer else None
|
||||
)
|
||||
self._slot_mapping_staging = None
|
||||
if self.is_host_writer:
|
||||
max_mirror_rows = (
|
||||
self.vllm_config.scheduler_config.max_num_batched_tokens
|
||||
+ max_num_reqs * (self.vllm_config.num_lookahead_tokens + 1)
|
||||
)
|
||||
self._slot_mapping_staging = _SlotMappingStaging(
|
||||
stream=torch.Stream(device=device),
|
||||
event=torch.Event(),
|
||||
slots=torch.empty(
|
||||
max_mirror_rows,
|
||||
dtype=torch.int64,
|
||||
pin_memory=True,
|
||||
),
|
||||
)
|
||||
self._dma_free_descriptors: list[_DMADescriptors] = []
|
||||
self._pending_dma_descriptors: deque[tuple[torch.Event, _DMADescriptors]] = (
|
||||
deque()
|
||||
)
|
||||
self._dma_submitted = False
|
||||
self._per_layer_mirrored: set[int] = set()
|
||||
self._submitted_mirror_layers: set[int] = set()
|
||||
self._layer_ready_events = tuple(torch.Event() for _ in cache_handles)
|
||||
self._forward_ready_event = torch.Event()
|
||||
self._draft_layers = tuple(
|
||||
index for index, cache in enumerate(cache_handles) if cache.draft_layer
|
||||
)
|
||||
self._draft_mirror_pending = False
|
||||
self._set_row_mirrors(())
|
||||
self.cache_layer_names = cache_layer_names
|
||||
self._group_leaders = tuple(
|
||||
(layer_name, cache)
|
||||
for layer_name, cache in zip(cache_layer_names, cache_handles, strict=True)
|
||||
if cache.runtime.is_group_leader
|
||||
)
|
||||
self.host_write_events = _create_hisparse_host_events(
|
||||
shared_host_region, is_host_writer, device
|
||||
)
|
||||
self.host_write_event = self.host_write_events[1]
|
||||
self._next_host_write_event = 0
|
||||
self.cache_handles = cache_handles
|
||||
self.leader_runtimes = [cache.runtime for _, cache in self._group_leaders]
|
||||
request_state_indices = {
|
||||
indices.data_ptr(): indices
|
||||
for cache in cache_handles
|
||||
if (indices := cache.runtime.request_state_indices) is not None
|
||||
}
|
||||
if len(request_state_indices) != 1:
|
||||
raise RuntimeError(
|
||||
"HiSparse runtimes must share one request-state mapping."
|
||||
)
|
||||
self.request_state_indices = next(iter(request_state_indices.values()))
|
||||
if self.request_state_indices.numel() != max_num_reqs:
|
||||
raise RuntimeError(
|
||||
"HiSparse request-state mapping does not match max_num_seqs."
|
||||
)
|
||||
self.hot_backing = hot_backing
|
||||
self.residency: torch.Tensor | None = next(
|
||||
(cache.residency for cache in cache_handles if cache.residency is not None),
|
||||
None,
|
||||
)
|
||||
self._pending_invalid_block_ids: list[int] = []
|
||||
# Destination block ids of host copies this worker has run.
|
||||
self._completed_host_copy_dst_ids: list[int] = []
|
||||
self._post_forward_transfers: list[SparseKVPageTransfer] = []
|
||||
self._enqueued_transfer_ids: list[int] = []
|
||||
self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = (
|
||||
deque()
|
||||
)
|
||||
self._metrics_calls = 0
|
||||
self._metrics_event = torch.Event()
|
||||
self._metrics_pending = False
|
||||
self._init_dma()
|
||||
if self.is_host_writer:
|
||||
for layer_index, handle in enumerate(cache_handles):
|
||||
# Draft layers are mirrored once in the next start_step instead.
|
||||
if not handle.draft_layer:
|
||||
handle.submit_layer_mirror = partial(
|
||||
self._enqueue_layer_mirror, layer_index
|
||||
)
|
||||
self._initialized = True
|
||||
|
||||
def set_request_state_indices(self, indices: torch.Tensor) -> None:
|
||||
if indices.numel() > self.request_state_indices.numel():
|
||||
raise ValueError(
|
||||
"HiSparse request-state mapping exceeds max_num_seqs: "
|
||||
f"{indices.numel()} > {self.request_state_indices.numel()}."
|
||||
)
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return
|
||||
# Attention indexes persistent request state by input-batch row. Refresh
|
||||
# that indirection after every batch compaction or reorder.
|
||||
self.request_state_indices.fill_(-1)
|
||||
self.request_state_indices[: indices.numel()].copy_(indices)
|
||||
if self._pending_invalid_block_ids:
|
||||
self.invalidate_blocks(self._pending_invalid_block_ids, indices)
|
||||
self._pending_invalid_block_ids.clear()
|
||||
|
||||
def _init_dma(self) -> None:
|
||||
host_caches = tuple(cache.runtime.host_cache for cache in self.cache_handles)
|
||||
resident_caches = []
|
||||
for cache in self.cache_handles:
|
||||
assert cache.view is not None and cache.slot_mapping is not None
|
||||
resident_caches.append(cache.view.cache)
|
||||
self.host_caches = host_caches
|
||||
self.resident_caches = tuple(resident_caches)
|
||||
|
||||
for resident_cache, host_cache in zip(resident_caches, host_caches):
|
||||
row_bytes = resident_cache.shape[-1] * resident_cache.element_size()
|
||||
if (
|
||||
resident_cache.ndim != 3
|
||||
or resident_cache.shape[1] != self.kernel_block_size
|
||||
or resident_cache.stride(1) * resident_cache.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError("HiSparse DMA requires contiguous resident rows.")
|
||||
if (
|
||||
host_cache.ndim != 2
|
||||
or not host_cache.is_contiguous()
|
||||
or host_cache.shape[1] * host_cache.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError("HiSparse DMA requires contiguous host rows.")
|
||||
|
||||
def start_step(
|
||||
self,
|
||||
metadata: HiSparseConnectorMetadata,
|
||||
request_state_indices: torch.Tensor | None,
|
||||
request_ids: list[str] | None = None,
|
||||
num_tokens: int = 0,
|
||||
) -> None:
|
||||
self._stage_row_mirror_mapping(num_tokens)
|
||||
self._finish_previous_step()
|
||||
previous_host_write_event = self.host_write_event
|
||||
self.host_write_event = self.host_write_events[self._next_host_write_event]
|
||||
self._next_host_write_event ^= 1
|
||||
current_stream().wait_event(previous_host_write_event)
|
||||
if self.shared_host_region is not None and not num_tokens:
|
||||
# Without a forward, no TP collective keeps rank 0 from recording
|
||||
# a shared event again before every rank has issued its wait.
|
||||
get_tp_group().barrier()
|
||||
self._release_completed_dma_descriptors()
|
||||
self._dma_submitted = False
|
||||
mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids)
|
||||
if self._slot_mapping_staging is not None:
|
||||
self._slot_mapping_staging.candidates = mirrors
|
||||
self._set_row_mirrors(mirrors)
|
||||
self._clear_forward_mirror_state()
|
||||
for handle in self.cache_handles:
|
||||
handle.all_context_pages_resident = metadata.all_context_pages_resident
|
||||
self._copy_host_blocks(metadata.host_block_copies, previous_host_write_event)
|
||||
transfers = (
|
||||
metadata.command.page_transfers if metadata.command is not None else []
|
||||
)
|
||||
self._restore_pages([transfer for transfer in transfers if transfer.restore])
|
||||
self._post_forward_transfers = [
|
||||
transfer
|
||||
for transfer in transfers
|
||||
if not transfer.restore and transfer.after_forward
|
||||
]
|
||||
self._submit_transfers(
|
||||
[
|
||||
transfer
|
||||
for transfer in transfers
|
||||
if not transfer.restore and not transfer.after_forward
|
||||
]
|
||||
)
|
||||
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
|
||||
if request_state_indices is not None:
|
||||
self.set_request_state_indices(request_state_indices)
|
||||
if metadata.residency_updates:
|
||||
assert self.residency is not None
|
||||
assert request_state_indices is not None and request_ids is not None
|
||||
update_hisparse_residency(
|
||||
self.residency,
|
||||
metadata.residency_updates,
|
||||
request_ids,
|
||||
request_state_indices,
|
||||
)
|
||||
|
||||
def _clear_forward_mirror_state(self) -> None:
|
||||
self._per_layer_mirrored.clear()
|
||||
self._submitted_mirror_layers.clear()
|
||||
for handle in self.cache_handles:
|
||||
handle.decode_batch = False
|
||||
handle.host_mirror_required = False
|
||||
handle.num_actual_tokens = 0
|
||||
handle.num_decode_tokens = 0
|
||||
handle.req_id_per_token = None
|
||||
|
||||
def prepare_forward(self, attn_metadata: Mapping[str, Any] | None) -> None:
|
||||
if attn_metadata is None:
|
||||
return
|
||||
for layer_name, handle in self._group_leaders:
|
||||
metadata = attn_metadata.get(layer_name)
|
||||
if metadata is not None:
|
||||
handle.prepare_group_for_batch(metadata)
|
||||
|
||||
def _stage_row_mirror_mapping(self, num_tokens: int) -> None:
|
||||
"""Snapshot the rows this forward will write, off the compute stream.
|
||||
|
||||
The resident slot mapping is a persistent view bound at registration
|
||||
time, so the rows are already staged by the time the forward launches.
|
||||
"""
|
||||
state = self._slot_mapping_staging
|
||||
if state is None or not num_tokens:
|
||||
return
|
||||
handle = self.cache_handles[0]
|
||||
slots = handle.slot_mapping
|
||||
assert slots is not None
|
||||
source_index = handle.runtime.resident_source_index
|
||||
start = state.num_tokens
|
||||
end = start + num_tokens
|
||||
if end > state.slots.shape[0]:
|
||||
raise ValueError(
|
||||
"HiSparse row mapping exceeds staging capacity: "
|
||||
f"{end} > {state.slots.shape[0]}."
|
||||
)
|
||||
main_stream = current_stream()
|
||||
state.stream.wait_stream(main_stream)
|
||||
with torch.cuda.stream(state.stream):
|
||||
state.slots[start:end].copy_(
|
||||
slots[:num_tokens],
|
||||
non_blocking=True,
|
||||
)
|
||||
state.event.record(state.stream)
|
||||
state.num_tokens = end
|
||||
state.source_index = source_index
|
||||
|
||||
def _resolve_row_mirrors(self, state: _SlotMappingStaging) -> None:
|
||||
"""Compute the final row mirrors from the staged GPU slot mapping."""
|
||||
self._set_row_mirrors(
|
||||
_select_written_row_mirrors(
|
||||
state.candidates,
|
||||
state.slots[: state.num_tokens].numpy(),
|
||||
state.source_index,
|
||||
)
|
||||
)
|
||||
state.num_tokens = 0
|
||||
|
||||
def _copy_host_blocks(
|
||||
self,
|
||||
host_block_copies: Sequence[KVCacheBlockCopy],
|
||||
previous_host_write_event: torch.Event,
|
||||
) -> None:
|
||||
if not host_block_copies:
|
||||
return
|
||||
self._completed_host_copy_dst_ids.extend(
|
||||
copy.dst_block_id for copy in host_block_copies
|
||||
)
|
||||
if self.shared_host_region is None or get_tensor_model_parallel_rank() == 0:
|
||||
if self.host_caches:
|
||||
previous_host_write_event.synchronize()
|
||||
copy_kv_cache_blocks_inplace(
|
||||
self.host_caches,
|
||||
self.host_num_blocks,
|
||||
host_block_copies,
|
||||
)
|
||||
if self.shared_host_region is not None:
|
||||
get_tp_group().barrier()
|
||||
|
||||
def invalidate_blocks(
|
||||
self, block_ids: list[int], request_state_indices: torch.Tensor
|
||||
) -> None:
|
||||
"""Invalidate recycled host slots in this worker's leader runtimes."""
|
||||
if not block_ids:
|
||||
return
|
||||
device = self.cache_handles[0].runtime.device
|
||||
staging = torch.tensor(block_ids, dtype=torch.int32, pin_memory=True)
|
||||
blocks = staging.to(device, dtype=torch.int32, non_blocking=True)
|
||||
offsets = torch.arange(self.kernel_block_size, dtype=torch.int32, device=device)
|
||||
slots = (blocks[:, None] * self.kernel_block_size + offsets[None, :]).flatten()
|
||||
sorted_slots = torch.sort(slots).values
|
||||
state_indices = request_state_indices.to(device=device, dtype=torch.long)
|
||||
for runtime in self.leader_runtimes:
|
||||
runtime.invalidate_sorted_slots(sorted_slots, state_indices)
|
||||
|
||||
def reset_hot_state(self) -> None:
|
||||
for runtime in self.leader_runtimes:
|
||||
runtime.reset_hot_state()
|
||||
|
||||
def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None:
|
||||
stats = None
|
||||
if self._metrics_pending and self._metrics_event.query():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
for runtime in self.leader_runtimes:
|
||||
group = runtime.index_group
|
||||
hits, misses = group.swap_stats_host.tolist()
|
||||
if hits or misses:
|
||||
stats.record_snapshot(hits, misses, misses * group.stats_row_bytes)
|
||||
self._metrics_pending = False
|
||||
if stats.is_empty():
|
||||
stats = None
|
||||
|
||||
self._metrics_calls += 1
|
||||
if (
|
||||
self._metrics_calls % _METRICS_INTERVAL == 0
|
||||
and not self._metrics_pending
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
compute_stream = current_stream()
|
||||
for runtime in self.leader_runtimes:
|
||||
group = runtime.index_group
|
||||
compute_stream.wait_stream(group.copy_stream)
|
||||
group.swap_stats_host.copy_(group.swap_stats, non_blocking=True)
|
||||
group.swap_stats.zero_()
|
||||
group.copy_stream.wait_stream(compute_stream)
|
||||
self._metrics_event.record()
|
||||
self._metrics_pending = True
|
||||
return stats
|
||||
|
||||
def _release_completed_dma_descriptors(self) -> None:
|
||||
pending = self._pending_dma_descriptors
|
||||
while pending and pending[0][0].query():
|
||||
_, descriptors = pending.popleft()
|
||||
self._dma_free_descriptors.append(descriptors)
|
||||
|
||||
def _acquire_dma_descriptors(self, size: int) -> _DMADescriptors:
|
||||
for index, descriptors in enumerate(self._dma_free_descriptors):
|
||||
if descriptors.src.numel() >= size:
|
||||
return self._dma_free_descriptors.pop(index)
|
||||
return _allocate_dma_descriptors(size)
|
||||
|
||||
def _submit_dma_descriptors(
|
||||
self,
|
||||
descriptors: _DMADescriptors,
|
||||
descriptor_count: int,
|
||||
transfer_ids: tuple[int, ...] = (),
|
||||
ready_event: torch.Event | None = None,
|
||||
) -> None:
|
||||
stream = self.dma_stream
|
||||
assert stream is not None
|
||||
if ready_event is None:
|
||||
stream.wait_stream(current_stream())
|
||||
else:
|
||||
stream.wait_event(ready_event)
|
||||
completion_event = torch.Event()
|
||||
with torch.cuda.stream(stream):
|
||||
ops.swap_blocks_batch(
|
||||
descriptors.src[:descriptor_count],
|
||||
descriptors.dst[:descriptor_count],
|
||||
descriptors.sizes[:descriptor_count],
|
||||
)
|
||||
self.host_write_event.record(stream)
|
||||
completion_event.record(stream)
|
||||
self._pending_dma_descriptors.append((completion_event, descriptors))
|
||||
self._dma_submitted = True
|
||||
if transfer_ids:
|
||||
self._pending_transfer_events.append((completion_event, transfer_ids))
|
||||
self._enqueued_transfer_ids.extend(transfer_ids)
|
||||
|
||||
def _set_row_mirrors(self, mirrors: tuple[SparseKVRowMirror, ...]) -> None:
|
||||
self._row_mirrors = mirrors
|
||||
self._row_mirror_destination_starts = np.fromiter(
|
||||
(mirror.destination_start for mirror in mirrors),
|
||||
dtype=np.int64,
|
||||
count=len(mirrors),
|
||||
)
|
||||
self._row_mirror_counts = np.fromiter(
|
||||
(mirror.num_rows for mirror in mirrors),
|
||||
dtype=np.int64,
|
||||
count=len(mirrors),
|
||||
)
|
||||
if mirrors:
|
||||
self._row_mirror_source_starts = np.asarray(
|
||||
[mirror.source_starts for mirror in mirrors], dtype=np.int64
|
||||
)
|
||||
if self._row_mirror_source_starts.ndim != 2:
|
||||
raise RuntimeError("HiSparse DMA source mappings must be rectangular.")
|
||||
else:
|
||||
self._row_mirror_source_starts = np.empty((0, 0), dtype=np.int64)
|
||||
self._row_mirror_num_rows = int(self._row_mirror_counts.sum())
|
||||
|
||||
def _enqueue_row_dma(
|
||||
self, layer_indices: Sequence[int], ready_event: torch.Event | None = None
|
||||
) -> None:
|
||||
if (
|
||||
not layer_indices
|
||||
or not self._row_mirror_num_rows
|
||||
or not self.is_host_writer
|
||||
):
|
||||
return
|
||||
mirrors = self._row_mirrors
|
||||
num_layers = len(layer_indices)
|
||||
descriptor_count = len(mirrors) * num_layers
|
||||
descriptors = self._acquire_dma_descriptors(descriptor_count)
|
||||
destination_starts = self._row_mirror_destination_starts
|
||||
row_counts = self._row_mirror_counts
|
||||
for descriptor_offset, layer_index in enumerate(layer_indices):
|
||||
cache = self.cache_handles[layer_index]
|
||||
source_index = cache.runtime.resident_source_index
|
||||
if source_index >= self._row_mirror_source_starts.shape[1]:
|
||||
raise RuntimeError("HiSparse row DMA source index is out of range.")
|
||||
source_rows = self._row_mirror_source_starts[:, source_index]
|
||||
source = self.resident_caches[layer_index]
|
||||
destination = self.host_caches[layer_index]
|
||||
row_bytes = source.shape[-1] * source.element_size()
|
||||
if (
|
||||
source.stride(1) * source.element_size() != row_bytes
|
||||
or destination.shape[1] * destination.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError("HiSparse row DMA requires contiguous rows.")
|
||||
source_blocks, source_row_offsets = np.divmod(
|
||||
source_rows, self.kernel_block_size
|
||||
)
|
||||
source_out_of_range = (
|
||||
np.any(source_blocks < 0)
|
||||
or np.any(source_blocks >= source.shape[0])
|
||||
or np.any(source_row_offsets + row_counts > self.kernel_block_size)
|
||||
)
|
||||
if source_out_of_range:
|
||||
raise RuntimeError("HiSparse row DMA source is out of range.")
|
||||
if np.any(destination_starts < 0) or np.any(
|
||||
destination_starts + row_counts > destination.shape[0]
|
||||
):
|
||||
raise RuntimeError("HiSparse row DMA index is out of range.")
|
||||
descriptor_slice = slice(descriptor_offset, descriptor_count, num_layers)
|
||||
descriptors.src_np[descriptor_slice] = (
|
||||
source.data_ptr()
|
||||
+ source_blocks * source.stride(0) * source.element_size()
|
||||
+ source_row_offsets * row_bytes
|
||||
)
|
||||
descriptors.dst_np[descriptor_slice] = (
|
||||
destination.data_ptr() + destination_starts * row_bytes
|
||||
)
|
||||
descriptors.sizes_np[descriptor_slice] = row_counts * row_bytes
|
||||
|
||||
self._submit_dma_descriptors(
|
||||
descriptors, descriptor_count, ready_event=ready_event
|
||||
)
|
||||
|
||||
def _enqueue_layer_mirror(self, layer_index: int) -> None:
|
||||
handle = self.cache_handles[layer_index]
|
||||
if not handle.host_mirror_required:
|
||||
return
|
||||
if layer_index in self._per_layer_mirrored:
|
||||
raise RuntimeError(f"HiSparse layer {layer_index} mirrored twice.")
|
||||
state = self._slot_mapping_staging
|
||||
if state is not None and state.num_tokens:
|
||||
# The staging stream waited on the compute stream before the
|
||||
# forward launched, so this only drains pre-forward work that is
|
||||
# already queued ahead of the layer kernels: no GPU bubble.
|
||||
state.event.synchronize()
|
||||
self._resolve_row_mirrors(state)
|
||||
self._per_layer_mirrored.add(layer_index)
|
||||
next_layer = layer_index + 1
|
||||
if (
|
||||
next_layer < len(self.cache_handles)
|
||||
and self.cache_handles[next_layer].runtime.resident_source_index
|
||||
== handle.runtime.resident_source_index
|
||||
):
|
||||
return
|
||||
ready_event = self._layer_ready_events[layer_index]
|
||||
ready_event.record()
|
||||
pending_layers = tuple(
|
||||
sorted(self._per_layer_mirrored - self._submitted_mirror_layers)
|
||||
)
|
||||
self._enqueue_row_dma(
|
||||
pending_layers,
|
||||
ready_event=ready_event,
|
||||
)
|
||||
self._submitted_mirror_layers.update(pending_layers)
|
||||
|
||||
def _record_transfer_completion(
|
||||
self,
|
||||
transfers: list[SparseKVPageTransfer],
|
||||
*,
|
||||
stream: torch.Stream | None = None,
|
||||
) -> None:
|
||||
if not transfers:
|
||||
return
|
||||
if stream is None:
|
||||
if not self.is_host_writer:
|
||||
return
|
||||
stream = self.dma_stream
|
||||
assert stream is not None
|
||||
completion_event = torch.Event()
|
||||
completion_event.record(stream)
|
||||
transfer_ids = tuple(transfer.transfer_id for transfer in transfers)
|
||||
self._pending_transfer_events.append((completion_event, transfer_ids))
|
||||
self._enqueued_transfer_ids.extend(transfer_ids)
|
||||
|
||||
def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
"""Restore imported tails on every rank before its forward or graph replay."""
|
||||
if not transfers:
|
||||
return
|
||||
for layer_index, cache in enumerate(self.cache_handles):
|
||||
source_index = cache.runtime.resident_source_index
|
||||
source = self.host_caches[layer_index]
|
||||
destination = self.resident_caches[layer_index]
|
||||
for transfer in transfers:
|
||||
host_start = transfer.host_block_id * self.kernel_block_size
|
||||
resident_block = transfer.resident_block_ids[source_index]
|
||||
# Each import pays one H2D page copy and enqueue per layer/rank
|
||||
# instead of tail prefill on D. Restore the prompt rows before
|
||||
# decode appends to this page; unused rows do not extend the
|
||||
# logical sequence.
|
||||
# cudaHostRegister pins this pool outside PyTorch's allocator,
|
||||
# so the sync checker cannot recognize it via is_pinned().
|
||||
with gpu_sync_allowed():
|
||||
destination[resident_block].copy_(
|
||||
source[host_start : host_start + self.kernel_block_size],
|
||||
non_blocking=True,
|
||||
)
|
||||
self._record_transfer_completion(transfers, stream=current_stream())
|
||||
|
||||
def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
if self.cache_handles[0].runtime.eager_host_mirror:
|
||||
self._record_transfer_completion(transfers)
|
||||
else:
|
||||
self._enqueue_transfers(transfers)
|
||||
|
||||
def _enqueue_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
if not transfers or not self.is_host_writer:
|
||||
return
|
||||
num_layers = len(self.cache_handles)
|
||||
descriptor_count = len(transfers) * num_layers
|
||||
descriptors = self._acquire_dma_descriptors(descriptor_count)
|
||||
destination_rows = np.fromiter(
|
||||
(transfer.host_block_id * self.kernel_block_size for transfer in transfers),
|
||||
dtype=np.int64,
|
||||
count=len(transfers),
|
||||
)
|
||||
source_blocks_by_transfer = np.asarray(
|
||||
[transfer.resident_block_ids for transfer in transfers], dtype=np.int64
|
||||
)
|
||||
if source_blocks_by_transfer.ndim != 2:
|
||||
raise RuntimeError(
|
||||
"HiSparse spill DMA source mappings must be rectangular."
|
||||
)
|
||||
for layer_index, cache in enumerate(self.cache_handles):
|
||||
source_index = cache.runtime.resident_source_index
|
||||
if source_index >= source_blocks_by_transfer.shape[1]:
|
||||
raise RuntimeError("HiSparse spill DMA source index is out of range.")
|
||||
source_blocks = source_blocks_by_transfer[:, source_index]
|
||||
source = self.resident_caches[layer_index]
|
||||
destination = self.host_caches[layer_index]
|
||||
if np.any(source_blocks < 0) or np.any(source_blocks >= source.shape[0]):
|
||||
raise RuntimeError("HiSparse spill DMA source is out of range.")
|
||||
if np.any(destination_rows < 0) or np.any(
|
||||
destination_rows + self.kernel_block_size > destination.shape[0]
|
||||
):
|
||||
raise RuntimeError("HiSparse spill DMA destination is out of range.")
|
||||
row_bytes = source.shape[-1] * source.element_size()
|
||||
descriptor_slice = slice(layer_index, descriptor_count, num_layers)
|
||||
descriptors.src_np[descriptor_slice] = (
|
||||
source.data_ptr()
|
||||
+ source_blocks * source.stride(0) * source.element_size()
|
||||
)
|
||||
descriptors.dst_np[descriptor_slice] = (
|
||||
destination.data_ptr() + destination_rows * row_bytes
|
||||
)
|
||||
descriptors.sizes_np[descriptor_slice] = self.kernel_block_size * row_bytes
|
||||
self._submit_dma_descriptors(
|
||||
descriptors,
|
||||
descriptor_count,
|
||||
transfer_ids=tuple(transfer.transfer_id for transfer in transfers),
|
||||
)
|
||||
|
||||
def _enqueue_host_mirror(
|
||||
self,
|
||||
ready_event: torch.Event | None = None,
|
||||
) -> None:
|
||||
active = [
|
||||
(index, handle)
|
||||
for index, handle in enumerate(self.cache_handles)
|
||||
if handle.num_actual_tokens != 0
|
||||
]
|
||||
if not active:
|
||||
return
|
||||
|
||||
def mirror_key(handle: HiSparseCacheHandle) -> tuple:
|
||||
slots = handle.mirror_slot_mapping
|
||||
return (
|
||||
handle.num_actual_tokens,
|
||||
handle.num_decode_tokens,
|
||||
handle.decode_batch,
|
||||
handle.host_mirror_required,
|
||||
handle.runtime.eager_host_mirror,
|
||||
None if slots is None else slots.data_ptr(),
|
||||
)
|
||||
|
||||
keys = {mirror_key(handle) for _, handle in active}
|
||||
if len(keys) > 1:
|
||||
raise RuntimeError(
|
||||
f"HiSparse cache layers disagree on mirror metadata: {list(keys)}."
|
||||
)
|
||||
cache = active[0][1]
|
||||
if not cache.host_mirror_required:
|
||||
return
|
||||
dst_slots = cache.mirror_slot_mapping
|
||||
if dst_slots is None:
|
||||
raise RuntimeError("HiSparse host mirror has no source slot mapping.")
|
||||
num_rows = min(cache.num_actual_tokens, dst_slots.numel())
|
||||
if num_rows == 0:
|
||||
return
|
||||
if self.is_host_writer:
|
||||
# The drafter has not written its layers' rows yet; the next
|
||||
# start_step mirrors them after it runs.
|
||||
self._draft_mirror_pending = bool(self._draft_layers)
|
||||
expected_layers = {index for index, _ in active}.difference(
|
||||
self._draft_layers
|
||||
)
|
||||
if self._per_layer_mirrored and self._per_layer_mirrored != expected_layers:
|
||||
raise RuntimeError(
|
||||
"HiSparse per-layer DMA did not mirror every active layer: "
|
||||
f"expected {sorted(expected_layers)}, got "
|
||||
f"{sorted(self._per_layer_mirrored)}."
|
||||
)
|
||||
pending_layers = tuple(
|
||||
sorted(expected_layers - self._submitted_mirror_layers)
|
||||
)
|
||||
if pending_layers:
|
||||
self._enqueue_row_dma(pending_layers, ready_event=ready_event)
|
||||
self._submitted_mirror_layers.update(pending_layers)
|
||||
assert cache.req_id_per_token is not None
|
||||
for _, handle in active:
|
||||
if handle.runtime.is_group_leader:
|
||||
handle.runtime.invalidate_written_slots(
|
||||
dst_slots[:num_rows],
|
||||
cache.req_id_per_token[:num_rows],
|
||||
)
|
||||
|
||||
def _finish_mirror_phase(self, ready_event: torch.Event | None = None) -> None:
|
||||
state = self._slot_mapping_staging
|
||||
if state is not None and state.num_tokens:
|
||||
# Measured free: blocking here costs no throughput (26.7 vs 27.6
|
||||
# gen tok/s) and lets every step mirror exactly the written rows.
|
||||
state.event.synchronize()
|
||||
self._resolve_row_mirrors(state)
|
||||
self._enqueue_host_mirror(ready_event)
|
||||
self._clear_forward_mirror_state()
|
||||
|
||||
def finish_forward(self) -> None:
|
||||
compute_stream = current_stream()
|
||||
self._forward_ready_event.record()
|
||||
self._finish_mirror_phase(self._forward_ready_event)
|
||||
if self.is_host_writer:
|
||||
if self._dma_submitted:
|
||||
compute_stream.wait_event(self.host_write_event)
|
||||
self._dma_submitted = False
|
||||
else:
|
||||
self.host_write_event.record(compute_stream)
|
||||
self._release_completed_dma_descriptors()
|
||||
|
||||
def _finish_previous_step(self) -> None:
|
||||
"""Mirror the previous step's draft-layer rows, then hand its pages over.
|
||||
|
||||
Runs before this step's work, so after the previous step's drafter.
|
||||
Post-forward transfers are ordered behind the draft mirror, so a page is
|
||||
never reported clean, and its resident block never released, before
|
||||
its draft-layer rows reach the host. Both land in the previous step's
|
||||
host write event, which this step waits on before its forward and its
|
||||
host copies, since the scheduler may already have reused the
|
||||
finished requests' blocks they read and write.
|
||||
"""
|
||||
if self._draft_mirror_pending:
|
||||
self._draft_mirror_pending = False
|
||||
self._enqueue_row_dma(self._draft_layers)
|
||||
transfers = self._post_forward_transfers
|
||||
self._post_forward_transfers = []
|
||||
self._submit_transfers(transfers)
|
||||
|
||||
def take_completed_host_copies(self) -> list[int]:
|
||||
"""Drain host copies this worker has enqueued for this step."""
|
||||
completed = self._completed_host_copy_dst_ids
|
||||
self._completed_host_copy_dst_ids = []
|
||||
return completed
|
||||
|
||||
def take_transfer_updates(self) -> tuple[list[int], list[int]]:
|
||||
enqueued = self._enqueued_transfer_ids
|
||||
self._enqueued_transfer_ids = []
|
||||
completed: list[int] = []
|
||||
while self._pending_transfer_events:
|
||||
event, transfer_ids = self._pending_transfer_events[0]
|
||||
if not event.query():
|
||||
break
|
||||
self._pending_transfer_events.popleft()
|
||||
completed.extend(transfer_ids)
|
||||
return enqueued, completed
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if not self._initialized:
|
||||
return
|
||||
if self._slot_mapping_staging is not None:
|
||||
self._slot_mapping_staging.stream.synchronize()
|
||||
if self.dma_stream is not None:
|
||||
self.dma_stream.synchronize()
|
||||
release_pinned_state(
|
||||
[cache.runtime for cache in self.cache_handles],
|
||||
self.pinned_host_pools,
|
||||
self.shared_host_region,
|
||||
)
|
||||
drop_hisparse_host_pool(self.vllm_config)
|
||||
self._initialized = False
|
||||
@@ -1,656 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
import msgspec
|
||||
import regex as re
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
from vllm.config import KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import (
|
||||
get_ip,
|
||||
get_open_port,
|
||||
make_zmq_socket,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
from dataclasses import field
|
||||
from enum import Enum
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
Transfer = tuple[int, float]
|
||||
EngineId = str
|
||||
ReqId = str
|
||||
TransferId = str
|
||||
TransferOffsetsKey = tuple[str, tuple[int, ...], tuple[int, ...], torch.dtype]
|
||||
|
||||
|
||||
def _positive_finite_timeout(name: str, value: Any) -> float:
|
||||
timeout = float(value)
|
||||
if not math.isfinite(timeout) or timeout <= 0:
|
||||
raise ValueError(f"{name} must be finite and greater than zero")
|
||||
return timeout
|
||||
|
||||
|
||||
class MoRIIOTransferAck(NamedTuple):
|
||||
transfer_id: TransferId
|
||||
consumer_tp_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class WriteTask:
|
||||
request_id: ReqId
|
||||
transfer_id: TransferId
|
||||
dst_engine_id: str
|
||||
local_block_ids: BlockIds
|
||||
remote_block_ids_hint: BlockIds | None
|
||||
layer_name: str
|
||||
event: torch.cuda.Event
|
||||
remote_notify_port: int
|
||||
remote_ip: str
|
||||
remote_tp_size: int | None = None
|
||||
multi_pod_hosts: list[str] = field(default_factory=list)
|
||||
remote_dp_size_local: int = 0
|
||||
remote_dp_size: int = 1
|
||||
enqueue_time: float = field(default_factory=time.perf_counter)
|
||||
retried: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class LayerTransferPlan:
|
||||
"""Plan for transferring a single layer."""
|
||||
|
||||
request_id: ReqId
|
||||
transfer_id: TransferId
|
||||
layer_name: str
|
||||
sess_idx: int
|
||||
transfer_local_offsets: list[int]
|
||||
transfer_remote_offsets: list[int]
|
||||
transfer_sizes: list[int]
|
||||
use_batch: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class RemoteAllocInfo:
|
||||
"""Information about remote block allocation."""
|
||||
|
||||
block_ids: BlockIds | None
|
||||
writes_done: int = 0
|
||||
writes_expected: int | None = None
|
||||
decode_dp_rank: int = 0
|
||||
completion_request_id: str | None = None
|
||||
completion_notify_port: int | None = None
|
||||
completion_remote_ip: str | None = None
|
||||
completion_notified: bool = False
|
||||
transfer_statuses: list[Any] = field(default_factory=list)
|
||||
transfer_offsets: dict[
|
||||
TransferOffsetsKey,
|
||||
tuple[list[int], list[int], list[int]],
|
||||
] = field(default_factory=dict)
|
||||
|
||||
|
||||
class ROLE(Enum):
|
||||
PRODUCER = "producer"
|
||||
CONSUMER = "consumer"
|
||||
NOTINIT = "notinit"
|
||||
|
||||
|
||||
class MoRIIOAgentMetadata(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
# required for @cached_property.d
|
||||
dict=True,
|
||||
):
|
||||
engine_id: str
|
||||
agent_metadata: bytes
|
||||
kv_caches_base_addr: list[int]
|
||||
num_blocks: int
|
||||
block_len: int
|
||||
attn_backend_name: str
|
||||
|
||||
|
||||
class RoleManager:
|
||||
"""Manages role state across the connector."""
|
||||
|
||||
_instance: "RoleManager | None" = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._role: ROLE = ROLE.NOTINIT
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls) -> "RoleManager":
|
||||
if cls._instance is None:
|
||||
with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
def set_role(self, role: ROLE) -> None:
|
||||
"""Set the current role."""
|
||||
with self._lock:
|
||||
self._role = role
|
||||
|
||||
def get_role(self) -> ROLE:
|
||||
"""Get the current role."""
|
||||
return self._role
|
||||
|
||||
|
||||
def set_role(role: ROLE):
|
||||
"""Set the global role."""
|
||||
RoleManager.get_instance().set_role(role)
|
||||
|
||||
|
||||
def get_role() -> ROLE:
|
||||
"""Get the global role."""
|
||||
return RoleManager.get_instance().get_role()
|
||||
|
||||
|
||||
class MoRIIOMode(Enum):
|
||||
READ = "read"
|
||||
WRITE = "write"
|
||||
|
||||
|
||||
class TransferBatchState(Enum):
|
||||
"""Verdict for a group of transfer statuses that belong to one request.
|
||||
|
||||
A request's KV transfer is spread over one status per layer (two for a KDA
|
||||
layer), so a single status never decides the request: PENDING means at
|
||||
least one is still in flight and none has failed, FAILED means at least
|
||||
one failed, DONE means all succeeded.
|
||||
"""
|
||||
|
||||
DONE = "done"
|
||||
FAILED = "failed"
|
||||
PENDING = "pending"
|
||||
|
||||
|
||||
class MoRIIOError(Exception):
|
||||
"""Base exception for MoRIIO operations."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class HandshakeError(MoRIIOError):
|
||||
"""Exception raised when handshake fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TransferError(MoRIIOError):
|
||||
"""Exception raised when transfer fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def get_moriio_mode(kv_transfer_config: KVTransferConfig) -> MoRIIOMode:
|
||||
read_mode = str(
|
||||
kv_transfer_config.kv_connector_extra_config.get("read_mode", "false")
|
||||
).lower().strip() in ("true", "1")
|
||||
logger.debug("MoRIIO Connector read_mode: %s", read_mode)
|
||||
if read_mode:
|
||||
return MoRIIOMode.READ
|
||||
else:
|
||||
return MoRIIOMode.WRITE
|
||||
|
||||
|
||||
def get_port_offset(dp_rank: int, tp_rank: int, tp_size: int = 1) -> int:
|
||||
return (dp_rank) * tp_size + tp_rank
|
||||
|
||||
|
||||
def fold_local_rank(global_dp_rank: int, dp_size_local: int) -> int:
|
||||
"""Fold a global DP rank into its pod-local rank [0, dp_size_local).
|
||||
|
||||
``dp_size_local == 0`` is the external-DP sentinel (local size unknown):
|
||||
return the rank unchanged, since a global DP rank is always < the global
|
||||
DP size so no folding is needed and the modulo is skipped (never divides
|
||||
by zero).
|
||||
"""
|
||||
return global_dp_rank % dp_size_local if dp_size_local else global_dp_rank
|
||||
|
||||
|
||||
def pod_index(global_dp_rank: int, dp_size_local: int) -> int:
|
||||
"""Pod index (0-based) a global DP rank lives on for Wide-EP multi-pod.
|
||||
|
||||
``dp_size_local == 0`` (external-DP sentinel) collapses to a single pod.
|
||||
"""
|
||||
return global_dp_rank // dp_size_local if dp_size_local else 0
|
||||
|
||||
|
||||
def resolve_host_ip(extra_config: dict) -> str:
|
||||
"""The IP this MoRIIO process advertises for KV transfer.
|
||||
|
||||
Honors an explicit ``host_ip`` in ``kv_connector_extra_config`` before
|
||||
falling back to ``get_ip()``. An external router/orchestrator can set it to
|
||||
the node's routable address; this is required under frameworks (e.g. Ray)
|
||||
where ``get_ip()`` resolves to an unroutable public IP and ``VLLM_HOST_IP``
|
||||
cannot be propagated to the worker processes that bind the transfer engine.
|
||||
"""
|
||||
return extra_config.get("host_ip") or get_ip()
|
||||
|
||||
|
||||
_DEPRECATED_ENV_VARS: dict[str, str] = {
|
||||
"VLLM_MORIIO_CONNECTOR_READ_MODE": "read_mode",
|
||||
"VLLM_MORIIO_QP_PER_TRANSFER": "qp_per_transfer",
|
||||
"VLLM_MORIIO_POST_BATCH_SIZE": "post_batch_size",
|
||||
"VLLM_MORIIO_NUM_WORKERS": "num_workers",
|
||||
}
|
||||
|
||||
|
||||
def _warn_deprecated_env_vars() -> None:
|
||||
for env_var, new_key in _DEPRECATED_ENV_VARS.items():
|
||||
if env_var in os.environ:
|
||||
logger.warning_once(
|
||||
"The environment variable %s is deprecated and ignored. "
|
||||
"Set %r inside kv_transfer_config.kv_connector_extra_config "
|
||||
"instead.",
|
||||
env_var,
|
||||
new_key,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MoRIIOConfig:
|
||||
local_ip: str
|
||||
local_kv_port: int
|
||||
proxy_ip: str
|
||||
local_ping_port: int
|
||||
proxy_ping_port: int
|
||||
http_port: int
|
||||
handshake_port: int
|
||||
notify_port: int
|
||||
tp_rank: int
|
||||
dp_rank: int
|
||||
dp_size: int
|
||||
tp_size: int
|
||||
transfer_timeout: float
|
||||
defer_timeout: float
|
||||
recv_abort_timeout: float
|
||||
read_mode: bool = False
|
||||
qp_per_transfer: int = 1
|
||||
post_batch_size: int = -1
|
||||
num_workers: int = 1
|
||||
backend: str = "rdma"
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(cls, vllm_config: VllmConfig) -> "MoRIIOConfig":
|
||||
# Port Configuration:
|
||||
# local_ping_port -> Outgoing heartbeat to proxy
|
||||
# proxy_ping_port -> Remote proxy's heartbeat ingress port
|
||||
# http_port -> Instance's HTTP service endpoint
|
||||
# local_kv_port -> service port for mori engine
|
||||
# notify_port -> For synchronizing stages between prefill and decode
|
||||
# handshake_port -> For initial handshake between mori engine
|
||||
|
||||
# Optional tuning knobs
|
||||
# read_mode -> If true, run the connector in READ mode (consumer
|
||||
# pulls KV from producer) instead of the default
|
||||
# WRITE mode.
|
||||
# transfer_timeout -> Timeout for waiting_for_transfer_complete before
|
||||
# raising TransferError (sec).
|
||||
# defer_timeout -> Timeout for deferred WRITE allocation or a missing
|
||||
# READ release notification (sec).
|
||||
# recv_abort_timeout -> Timeout before an in-flight recv whose RDMA
|
||||
# completion never arrived is aborted (sec).
|
||||
|
||||
# Knobs for RDMA transfers, ignored if on xgmi backend
|
||||
# qp_per_transfer -> Number of RDMA Queue Pairs per KV transfer.
|
||||
# post_batch_size -> Batch size for posting transfer work requests
|
||||
# (-1 lets the MoRI backend choose).
|
||||
# num_workers -> Number of background worker threads the MoRI
|
||||
# engine uses for transfer processing.
|
||||
|
||||
# TODO : merge notify_port and handshake_port to simplify port management
|
||||
# supports non-contiguous ports
|
||||
assert vllm_config.kv_transfer_config is not None, (
|
||||
"kv_transfer_config must be set for MoRIIOConnector"
|
||||
)
|
||||
_warn_deprecated_env_vars()
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
# Fold the global data_parallel_rank back to [0, dp_size_local) for
|
||||
# per-node port allocation (handles the external-DP sentinel).
|
||||
pc = vllm_config.parallel_config
|
||||
dp_rank = fold_local_rank(pc.data_parallel_rank, pc.data_parallel_size_local)
|
||||
base_notify_port = int(extra_config["notify_port"])
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
port_offset = get_port_offset(dp_rank, tp_rank)
|
||||
backend = str(extra_config.get("backend", "rdma")).lower()
|
||||
if backend not in ("rdma", "xgmi"):
|
||||
raise ValueError(
|
||||
f"Invalid MoRIIO backend {backend!r} in kv_connector_extra_config; "
|
||||
"must be one of 'rdma' or 'xgmi'."
|
||||
)
|
||||
|
||||
transfer_timeout = _positive_finite_timeout(
|
||||
"kv_connector_extra_config.transfer_timeout",
|
||||
extra_config.get(
|
||||
"transfer_timeout", MoRIIOConstants.DEFAULT_TRANSFER_TIMEOUT
|
||||
),
|
||||
)
|
||||
defer_timeout = _positive_finite_timeout(
|
||||
"kv_connector_extra_config.defer_timeout",
|
||||
extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT),
|
||||
)
|
||||
legacy_recv_abort_timeout = os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S")
|
||||
if legacy_recv_abort_timeout is not None:
|
||||
logger.warning_once(
|
||||
"The environment variable VLLM_MORIIO_TRANSFER_TIMEOUT_S is "
|
||||
"deprecated. Set 'recv_abort_timeout' inside "
|
||||
"kv_transfer_config.kv_connector_extra_config instead."
|
||||
)
|
||||
recv_abort_timeout = _positive_finite_timeout(
|
||||
"kv_connector_extra_config.recv_abort_timeout",
|
||||
extra_config.get(
|
||||
"recv_abort_timeout",
|
||||
legacy_recv_abort_timeout or MoRIIOConstants.DEFAULT_RECV_ABORT_TIMEOUT,
|
||||
),
|
||||
)
|
||||
|
||||
return cls(
|
||||
local_ip=resolve_host_ip(extra_config),
|
||||
local_kv_port=get_open_port(),
|
||||
proxy_ip=extra_config.get("proxy_ip", ""),
|
||||
local_ping_port=get_open_port(),
|
||||
proxy_ping_port=int(extra_config.get("proxy_ping_port", 0)),
|
||||
http_port=int(extra_config["http_port"]),
|
||||
handshake_port=int(extra_config["handshake_port"]),
|
||||
notify_port=base_notify_port + port_offset,
|
||||
tp_rank=tp_rank,
|
||||
dp_rank=dp_rank,
|
||||
# Advertise the GLOBAL DP world size to the routing proxy so it can
|
||||
# address every rank across pods; dp_size_local above is only for
|
||||
# per-pod port folding, not for routing.
|
||||
dp_size=pc.data_parallel_size,
|
||||
tp_size=tp_size,
|
||||
read_mode=get_moriio_mode(kv_transfer_config) == MoRIIOMode.READ,
|
||||
qp_per_transfer=int(extra_config.get("qp_per_transfer", 1)),
|
||||
post_batch_size=int(extra_config.get("post_batch_size", -1)),
|
||||
num_workers=int(extra_config.get("num_workers", 1)),
|
||||
backend=backend,
|
||||
transfer_timeout=transfer_timeout,
|
||||
defer_timeout=defer_timeout,
|
||||
recv_abort_timeout=recv_abort_timeout,
|
||||
)
|
||||
|
||||
|
||||
class MoRIIOConstants:
|
||||
"""Constants for MoRIIO connector."""
|
||||
|
||||
# ZMQ message types
|
||||
GET_META_MSG = b"get_meta_msg"
|
||||
POP_DONE_RECV = b"pop_done_recv"
|
||||
OVER = b"OVER"
|
||||
COMPLETION_PREFIX = "cmpl"
|
||||
TRANSFER_PREFIX = "tx"
|
||||
|
||||
PING_INTERVAL = 3
|
||||
MAX_PING_RETRIES = 100
|
||||
DEFAULT_HANDSHAKE_PORT = "6301"
|
||||
DEFAULT_NOTIFY_PORT = "61005"
|
||||
|
||||
VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT = 3600
|
||||
|
||||
# Timeout (seconds) for waiting_for_transfer_complete before raising TransferError.
|
||||
# Overridable via kv_connector_extra_config["transfer_timeout"].
|
||||
DEFAULT_TRANSFER_TIMEOUT = 30.0
|
||||
# Timeout (seconds) for deferred WRITE allocation or a missing READ release
|
||||
# notification.
|
||||
# Overridable via kv_connector_extra_config["defer_timeout"].
|
||||
DEFAULT_DEFER_TIMEOUT = 60.0
|
||||
# Timeout (seconds) before an in-flight recv whose RDMA completion was lost
|
||||
# is aborted, so the decode worker does not hang on it forever.
|
||||
# Overridable via kv_connector_extra_config["recv_abort_timeout"].
|
||||
DEFAULT_RECV_ABORT_TIMEOUT = 120.0
|
||||
|
||||
|
||||
# The router embeds both zmq_addresses in the request_id:
|
||||
# "___prefill_addr_{zmq}___decode_addr_{zmq}_{32-hex-uuid}"
|
||||
# MoRIIO zmq_address format: "host:IP,handshake:PORT,notify:PORT"
|
||||
#
|
||||
# This lets each connector side parse the peer's connection info without
|
||||
# requiring the router to pass it explicitly in kv_transfer_params.
|
||||
_PREFILL_ZMQ_RE = re.compile(r"___prefill_addr_(.+?)___decode_addr_")
|
||||
# vLLM wraps the router's X-Request-Id as "cmpl-<id>-<seq>-<hex>" so there may
|
||||
# be a trailing "-<seq>-<hex>" suffix after the 32-char UUID. Allow it.
|
||||
_DECODE_ZMQ_RE = re.compile(r"___decode_addr_(.+)_[0-9a-f]{32}(?:-.*)?$")
|
||||
|
||||
|
||||
def parse_moriio_zmq_address(
|
||||
zmq_address: str,
|
||||
) -> tuple[str, int, int]:
|
||||
"""Parse the MoRI-IO zmq address into its components.
|
||||
|
||||
Parses ``"host:IP,handshake:PORT,notify:PORT"`` into
|
||||
(host, handshake_port, notify_port).
|
||||
|
||||
Each key-value pair is split on the *first* colon so that IPv6 addresses
|
||||
(e.g. ``host:::1``) are handled correctly. Raises ``ValueError`` if any
|
||||
of ``host``, ``handshake``, or ``notify`` keys are absent or if the port
|
||||
values are non-numeric.
|
||||
"""
|
||||
parts: dict[str, str] = {}
|
||||
for segment in zmq_address.split(","):
|
||||
key, _, val = segment.partition(":")
|
||||
parts[key.strip()] = val.strip()
|
||||
try:
|
||||
host = parts["host"]
|
||||
handshake_port = int(parts["handshake"])
|
||||
notify_port = int(parts["notify"])
|
||||
except (KeyError, ValueError) as e:
|
||||
raise ValueError(
|
||||
f"Malformed zmq_address {zmq_address!r}: expected "
|
||||
f"'host:IP,handshake:PORT,notify:PORT' format"
|
||||
) from e
|
||||
return host, handshake_port, notify_port
|
||||
|
||||
|
||||
def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str | None:
|
||||
"""Extract the *peer's* zmq_address from the vLLM router request_id.
|
||||
|
||||
The producer (prefill) needs the decode's address; the consumer (decode)
|
||||
needs the prefill's address.
|
||||
|
||||
Returns ``None`` when the request_id does not encode peer info. The
|
||||
llm-d routing sidecar (``llm-d-inference-scheduler``) does not embed
|
||||
addresses in ``request_id``; instead it passes ``remote_host``,
|
||||
``remote_handshake_port`` and ``remote_notify_port`` explicitly in
|
||||
``kv_transfer_params``. Callers must handle the ``None`` return by
|
||||
falling back to those fields. See ``add_new_req`` for the canonical
|
||||
fallback path.
|
||||
"""
|
||||
if is_producer:
|
||||
m = _DECODE_ZMQ_RE.search(request_id)
|
||||
else:
|
||||
m = _PREFILL_ZMQ_RE.search(request_id)
|
||||
if m is None:
|
||||
return None
|
||||
return m.group(1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
"""Metadata for a single request."""
|
||||
|
||||
transfer_id: TransferId
|
||||
local_block_ids: BlockIds
|
||||
remote_block_ids: BlockIds
|
||||
remote_host: str
|
||||
remote_port: int
|
||||
remote_handshake_port: int
|
||||
remote_notify_port: int
|
||||
remote_engine_id: str
|
||||
tp_size: int
|
||||
remote_dp_size: int
|
||||
# Prefill DP rank that owns this request's KV (forwarded by the proxy). The
|
||||
# read must target this rank's memory registration; the default 0 preserves
|
||||
# the symmetric single-DP behaviour.
|
||||
remote_dp_rank: int = 0
|
||||
# Multi-pod: list of remote pod IPs indexed by pod_idx.
|
||||
multi_pod_hosts: list[str] = field(default_factory=list)
|
||||
# Per-pod DP size; 0 means fallback to remote_dp_size.
|
||||
remote_dp_size_local: int = 0
|
||||
|
||||
|
||||
class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
def __init__(self):
|
||||
self.reqs_to_recv: dict[ReqId, ReqMeta] = {}
|
||||
self.reqs_to_save: dict[ReqId, ReqMeta] = {}
|
||||
self.reqs_to_send: dict[ReqId, float] = {}
|
||||
self.transfer_id_to_request_id: dict[TransferId, ReqId] = {}
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"MoRIIOConnectorMetadata: reqs_to_recv={self.reqs_to_recv}, "
|
||||
f"reqs_to_save={self.reqs_to_save}, "
|
||||
f"reqs_to_send={self.reqs_to_send}, "
|
||||
f"transfer_id_to_request_id={self.transfer_id_to_request_id}"
|
||||
)
|
||||
|
||||
def add_new_req(
|
||||
self,
|
||||
request_id: ReqId,
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
write_mode=False,
|
||||
):
|
||||
"""Ingest a peer's ``kv_transfer_params`` into a typed ``ReqMeta``.
|
||||
|
||||
This is the single place peer info enters the connector. The
|
||||
``kv_transfer_params`` contract (as produced by the llm-d sidecar /
|
||||
vLLM router, or echoed by the prefill leg's ``request_finished``):
|
||||
|
||||
Required (always):
|
||||
* ``transfer_id`` -- stable id shared by both legs.
|
||||
* ``remote_engine_id`` -- peer engine id for the handshake table.
|
||||
* ``remote_block_ids`` -- peer block ids (may be [] in WRITE mode,
|
||||
where decode allocates its own blocks).
|
||||
|
||||
Peer address -- ONE of the following two must resolve:
|
||||
* embedded in ``request_id`` (vLLM-router PD id form), OR
|
||||
* explicit ``remote_host`` + ``remote_handshake_port`` +
|
||||
``remote_notify_port`` (llm-d sidecar / returnable path).
|
||||
If neither resolves we raise -- there is no safe default host/port.
|
||||
|
||||
Optional (defaulted):
|
||||
* ``tp_size`` (default 1) -- peer TP size.
|
||||
* ``remote_dp_size`` (default 1) -- peer GLOBAL DP size.
|
||||
* ``remote_dp_size_local`` (default = ``remote_dp_size``) -- per-pod
|
||||
DP size for Wide-EP multi-pod port/host
|
||||
folding; 0/absent means single-pod.
|
||||
* ``remote_hosts`` (default [remote_host]) -- per-pod IP list
|
||||
indexed by ``pod_idx = rank // dp_local``.
|
||||
|
||||
Routing keys consumed elsewhere (NOT here): ``remote_dp_rank`` /
|
||||
``remote_dp_rank_override`` gate the decode->prefill notify target in
|
||||
MoRIIOConnectorScheduler; they are router-authoritative and never
|
||||
self-derived (see that class's request routing contract).
|
||||
"""
|
||||
transfer_id = kv_transfer_params["transfer_id"]
|
||||
|
||||
# Try request_id embedded address first, fallback to explicit params.
|
||||
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
|
||||
if peer_zmq is not None:
|
||||
remote_host, remote_handshake_port, remote_notify_port = (
|
||||
parse_moriio_zmq_address(peer_zmq)
|
||||
)
|
||||
else:
|
||||
try:
|
||||
remote_host = kv_transfer_params["remote_host"]
|
||||
if not remote_host:
|
||||
raise ValueError(
|
||||
f"request_id {request_id!r} does not embed a peer "
|
||||
f"zmq_address and kv_transfer_params['remote_host'] is "
|
||||
f"empty; cannot route MoRI-IO transfer"
|
||||
)
|
||||
remote_handshake_port = int(kv_transfer_params["remote_handshake_port"])
|
||||
remote_notify_port = int(kv_transfer_params["remote_notify_port"])
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
raise ValueError(
|
||||
f"request_id {request_id!r} does not embed a peer "
|
||||
f"zmq_address and kv_transfer_params is missing one or "
|
||||
f"more sidecar-fallback keys (need remote_host, "
|
||||
f"remote_handshake_port, remote_notify_port): {e}"
|
||||
) from e
|
||||
|
||||
# Multi-pod: use multi_pod_hosts list or fallback to single host.
|
||||
_pod_hosts = kv_transfer_params.get("remote_hosts") or [remote_host]
|
||||
if not isinstance(_pod_hosts, list):
|
||||
_pod_hosts = [_pod_hosts]
|
||||
_pod_hosts = [str(h) for h in _pod_hosts]
|
||||
_remote_dp_size_local = int(
|
||||
kv_transfer_params.get(
|
||||
"remote_dp_size_local",
|
||||
kv_transfer_params.get("remote_dp_size", 1),
|
||||
)
|
||||
)
|
||||
|
||||
_req = ReqMeta(
|
||||
transfer_id=transfer_id,
|
||||
local_block_ids=local_block_ids,
|
||||
remote_block_ids=kv_transfer_params["remote_block_ids"],
|
||||
remote_engine_id=kv_transfer_params["remote_engine_id"],
|
||||
remote_host=remote_host,
|
||||
remote_port=int(remote_handshake_port),
|
||||
remote_handshake_port=int(remote_handshake_port),
|
||||
remote_notify_port=int(remote_notify_port),
|
||||
# Remote peer TP degree (used as remote_tp_size downstream). The
|
||||
# decode side must know the prefiller's TP degree to map producer
|
||||
# ranks onto the correct decode rank (and vice versa). The old plain
|
||||
# "tp_size" key (#46332) is absent on WRITE producer requests and
|
||||
# defaulted to 1, collapsing every producer rank onto decode rank 0
|
||||
# and corrupting the transfer; prefer "remote_tp_size". 0 == unknown
|
||||
# (treated as homogeneous downstream).
|
||||
tp_size=int(
|
||||
kv_transfer_params.get("remote_tp_size")
|
||||
or kv_transfer_params.get("tp_size")
|
||||
or 0
|
||||
),
|
||||
remote_dp_size=kv_transfer_params.get("remote_dp_size", 1),
|
||||
remote_dp_rank=kv_transfer_params.get("remote_dp_rank", 0),
|
||||
multi_pod_hosts=_pod_hosts,
|
||||
remote_dp_size_local=_remote_dp_size_local,
|
||||
)
|
||||
if write_mode:
|
||||
self.reqs_to_save[request_id] = _req
|
||||
else:
|
||||
self.reqs_to_recv[request_id] = _req
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
|
||||
"""Context manager for a ZMQ socket."""
|
||||
if socket_type not in (zmq.ROUTER, zmq.REQ, zmq.DEALER):
|
||||
raise ValueError(f"Unexpected socket type: {socket_type}")
|
||||
|
||||
ctx: zmq.Context | None = None
|
||||
try:
|
||||
ctx = zmq.Context() # type: ignore[attr-defined]
|
||||
yield make_zmq_socket(
|
||||
ctx=ctx, path=addr, socket_type=socket_type, bind=socket_type == zmq.ROUTER
|
||||
)
|
||||
finally:
|
||||
if ctx is not None:
|
||||
ctx.destroy(linger=0)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,96 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Keep discovery heartbeats independent of the model worker's GIL."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import msgpack
|
||||
import zmq
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MoRIIOHeartbeat:
|
||||
def __init__(
|
||||
self,
|
||||
endpoint: str,
|
||||
payload: dict[str, Any],
|
||||
interval: float,
|
||||
max_retries: int = 100,
|
||||
):
|
||||
self._process = subprocess.Popen(
|
||||
[sys.executable, __file__, endpoint, str(interval), str(max_retries)],
|
||||
stdin=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
assert self._process.stdin is not None
|
||||
try:
|
||||
self._process.stdin.write(json.dumps(payload) + "\n")
|
||||
self._process.stdin.flush()
|
||||
except BaseException:
|
||||
self.shutdown()
|
||||
raise
|
||||
|
||||
def check_health(self) -> None:
|
||||
returncode = self._process.poll()
|
||||
if returncode is not None:
|
||||
raise RuntimeError(
|
||||
f"MoRIIO discovery heartbeat exited with code {returncode}"
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
assert self._process.stdin is not None
|
||||
try:
|
||||
if not self._process.stdin.closed:
|
||||
self._process.stdin.close()
|
||||
except BrokenPipeError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
self._process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._process.terminate()
|
||||
try:
|
||||
self._process.wait(timeout=1)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._process.kill()
|
||||
self._process.wait(timeout=1)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
endpoint, interval_arg, max_retries_arg = sys.argv[1:]
|
||||
initial = sys.stdin.readline()
|
||||
if not initial:
|
||||
return
|
||||
payload = msgpack.dumps(json.loads(initial))
|
||||
interval = float(interval_arg)
|
||||
max_retries = int(max_retries_arg)
|
||||
failures = 0
|
||||
with zmq.Context() as context, context.socket(zmq.DEALER) as sock:
|
||||
sock.setsockopt(zmq.LINGER, 0)
|
||||
sock.setsockopt(zmq.SNDTIMEO, 1000)
|
||||
sock.connect(endpoint)
|
||||
while not select.select([sys.stdin], [], [], 0)[0]:
|
||||
try:
|
||||
sock.send(payload)
|
||||
failures = 0
|
||||
except zmq.Again:
|
||||
failures = 0
|
||||
logger.warning("MoRIIO discovery heartbeat send failed; retrying")
|
||||
except zmq.ZMQError:
|
||||
failures += 1
|
||||
logger.exception("MoRIIO discovery heartbeat send failed")
|
||||
if failures >= max_retries:
|
||||
raise
|
||||
# EOF also stops the child when the worker exits without cleanup.
|
||||
if select.select([sys.stdin], [], [], interval)[0]:
|
||||
break
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,615 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import NamedTuple, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
|
||||
MambaConvSplitInfo,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
|
||||
class LayerTransferGeometry(NamedTuple):
|
||||
num_blocks: int
|
||||
block_size: int
|
||||
block_len: int
|
||||
slot_size_bytes: int
|
||||
block_stride: int
|
||||
local_kv_stride: int | None
|
||||
remote_kv_stride: int | None
|
||||
transfers_per_block: int
|
||||
regions_per_block: int
|
||||
split_kv_regions: bool
|
||||
|
||||
|
||||
class MambaTransferGeometry(NamedTuple):
|
||||
"""Geometry of a hybrid (mamba/KDA) layer's recurrent state.
|
||||
|
||||
A KDA layer's packed page is exposed as ``conv_state`` and ``ssm_state``
|
||||
views. Each is slot-strided, so a slot's bytes live at
|
||||
``slot * slot_stride * element_size`` and span ``slot_bytes``. The two
|
||||
tensors are registered as two separate MoRIIO regions per layer.
|
||||
"""
|
||||
|
||||
conv_slot_stride: int
|
||||
ssm_slot_stride: int
|
||||
ssm_slot_bytes: int
|
||||
conv_element_size: int
|
||||
ssm_element_size: int
|
||||
conv_region_len: int
|
||||
ssm_region_len: int
|
||||
|
||||
|
||||
def get_mamba_transfer_geometry(
|
||||
conv: torch.Tensor,
|
||||
ssm: torch.Tensor,
|
||||
) -> MambaTransferGeometry:
|
||||
"""Derive the slot-strided geometry of a KDA layer's conv + ssm state.
|
||||
|
||||
``conv``/``ssm`` are zero-copy views over the layer's packed page. Slot byte
|
||||
offset is ``slot * slot_stride * element_size`` and each
|
||||
slot spans ``subtensor[0].numel() * element_size`` bytes; the whole tensor
|
||||
is registered as one region (``*_region_len`` bytes).
|
||||
"""
|
||||
if conv.ndim == 0 or ssm.ndim == 0 or conv.shape[0] == 0 or ssm.shape[0] == 0:
|
||||
raise ValueError("KDA conv/ssm caches must contain at least one slot")
|
||||
if conv.shape[0] != ssm.shape[0]:
|
||||
raise ValueError(
|
||||
f"KDA conv/ssm slot count mismatch: {conv.shape[0]} != {ssm.shape[0]}"
|
||||
)
|
||||
conv_slot_bytes = conv[0].numel() * conv.element_size()
|
||||
ssm_slot_bytes = ssm[0].numel() * ssm.element_size()
|
||||
conv_available_bytes = (
|
||||
conv.untyped_storage().nbytes() - conv.storage_offset() * conv.element_size()
|
||||
)
|
||||
ssm_available_bytes = (
|
||||
ssm.untyped_storage().nbytes() - ssm.storage_offset() * ssm.element_size()
|
||||
)
|
||||
if conv.shape[0]:
|
||||
conv_required_bytes = (conv.shape[0] - 1) * conv.stride(
|
||||
0
|
||||
) * conv.element_size() + conv_slot_bytes
|
||||
ssm_required_bytes = (ssm.shape[0] - 1) * ssm.stride(
|
||||
0
|
||||
) * ssm.element_size() + ssm_slot_bytes
|
||||
if conv_required_bytes > conv_available_bytes:
|
||||
raise ValueError("KDA conv slots exceed the registered storage extent")
|
||||
if ssm_required_bytes > ssm_available_bytes:
|
||||
raise ValueError("KDA ssm slots exceed the registered storage extent")
|
||||
|
||||
return MambaTransferGeometry(
|
||||
conv_slot_stride=conv.stride(0),
|
||||
ssm_slot_stride=ssm.stride(0),
|
||||
ssm_slot_bytes=ssm_slot_bytes,
|
||||
conv_element_size=conv.element_size(),
|
||||
ssm_element_size=ssm.element_size(),
|
||||
# Registered region must span the full slot-strided extent: stride(0)
|
||||
# may exceed the per-slot element count, so numel() would under-size the
|
||||
# region and truncate the highest-slot transfers. shape[0]*stride(0)
|
||||
# covers every byte a ``slot * slot_stride * elem`` offset can address.
|
||||
# Cap at the bytes remaining from each view's storage offset. A packed
|
||||
# conv+ssm page places ssm at a non-zero intra-buffer offset,
|
||||
# so do not register past the end of the shared backing allocation.
|
||||
conv_region_len=min(
|
||||
conv.shape[0] * conv.stride(0) * conv.element_size(),
|
||||
conv_available_bytes,
|
||||
),
|
||||
ssm_region_len=min(
|
||||
ssm.shape[0] * ssm.stride(0) * ssm.element_size(),
|
||||
ssm_available_bytes,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def build_layer_to_spec(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheSpec]:
|
||||
layer_to_spec: dict[str, KVCacheSpec] = {}
|
||||
for group in kv_cache_config.kv_cache_groups:
|
||||
group_spec = group.kv_cache_spec
|
||||
if isinstance(group_spec, UniformTypeKVCacheSpecs):
|
||||
layer_to_spec.update(
|
||||
{
|
||||
layer_name: group_spec.kv_cache_specs[layer_name]
|
||||
for layer_name in group.layer_names
|
||||
}
|
||||
)
|
||||
else:
|
||||
layer_to_spec.update(
|
||||
{layer_name: group_spec for layer_name in group.layer_names}
|
||||
)
|
||||
return layer_to_spec
|
||||
|
||||
|
||||
def is_mla_cache_layer(
|
||||
layer_to_spec: Mapping[str, KVCacheSpec], layer_name: str
|
||||
) -> bool:
|
||||
try:
|
||||
spec = layer_to_spec[layer_name]
|
||||
except KeyError as e:
|
||||
raise ValueError(f"Missing KV cache spec for layer {layer_name}") from e
|
||||
return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
|
||||
|
||||
|
||||
def _spec_dim_matches(value: int, expected: int | None) -> bool:
|
||||
return expected is None or value == expected
|
||||
|
||||
|
||||
def _kernel_layout_matches(
|
||||
spec: KVCacheSpec, kernel_block_size: int, num_kv_heads: int, head_dim: int
|
||||
) -> bool:
|
||||
if kernel_block_size <= 0 or spec.block_size % kernel_block_size != 0:
|
||||
return False
|
||||
return _spec_dim_matches(
|
||||
num_kv_heads, getattr(spec, "num_kv_heads", None)
|
||||
) and _spec_dim_matches(head_dim, getattr(spec, "head_size", None))
|
||||
|
||||
|
||||
def _select_kernel_block_layout(
|
||||
layer_name: str, shape: torch.Size, spec: KVCacheSpec
|
||||
) -> tuple[int, int, int]:
|
||||
axis2_matches = _kernel_layout_matches(spec, shape[2], shape[3], shape[4])
|
||||
axis3_matches = _kernel_layout_matches(spec, shape[3], shape[2], shape[4])
|
||||
|
||||
if axis2_matches and axis3_matches and shape[2] != shape[3]:
|
||||
raise ValueError(
|
||||
f"Ambiguous MoRIIO kernel-block K/V cache shape for layer "
|
||||
f"{layer_name}: {tuple(shape)}"
|
||||
)
|
||||
if axis2_matches:
|
||||
return shape[2], shape[3], shape[4]
|
||||
if axis3_matches:
|
||||
return shape[3], shape[2], shape[4]
|
||||
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
|
||||
f"{tuple(shape)} does not contain block size {spec.block_size}"
|
||||
)
|
||||
|
||||
|
||||
def get_layer_transfer_geometry(
|
||||
layer_name: str,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_to_spec: Mapping[str, KVCacheSpec],
|
||||
remote_num_blocks: int | None = None,
|
||||
) -> LayerTransferGeometry:
|
||||
spec = layer_to_spec[layer_name]
|
||||
shape = kv_cache.shape
|
||||
stride = kv_cache.stride()
|
||||
element_size = kv_cache.element_size()
|
||||
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
|
||||
|
||||
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
|
||||
_, num_blocks = shape[:2]
|
||||
kernel_blocks_per_block = 1
|
||||
if shape[2] == spec.block_size:
|
||||
block_size, num_kv_heads, head_dim = shape[2:]
|
||||
elif shape[3] == spec.block_size:
|
||||
num_kv_heads, block_size, head_dim = shape[2:]
|
||||
else:
|
||||
kernel_num_blocks = num_blocks
|
||||
kernel_block_size, num_kv_heads, head_dim = _select_kernel_block_layout(
|
||||
layer_name, shape, spec
|
||||
)
|
||||
kernel_blocks_per_block = spec.block_size // kernel_block_size
|
||||
if kernel_num_blocks % kernel_blocks_per_block != 0:
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
|
||||
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
|
||||
f"not divisible by {kernel_blocks_per_block}"
|
||||
)
|
||||
num_blocks = kernel_num_blocks // kernel_blocks_per_block
|
||||
block_size = spec.block_size
|
||||
slot_size_bytes = num_kv_heads * head_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[1] * kernel_blocks_per_block,
|
||||
local_kv_stride=stride[0],
|
||||
remote_kv_stride=(
|
||||
stride[1] * kernel_blocks_per_block * (remote_num_blocks or num_blocks)
|
||||
),
|
||||
transfers_per_block=2,
|
||||
regions_per_block=1,
|
||||
split_kv_regions=True,
|
||||
)
|
||||
|
||||
if not is_mla_cache and len(shape) == 5 and shape[1] == 2:
|
||||
num_blocks = shape[0]
|
||||
if shape[2] == spec.block_size:
|
||||
block_size, num_kv_heads, head_dim = shape[2:]
|
||||
slot_size_bytes = num_kv_heads * head_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[0],
|
||||
local_kv_stride=stride[1],
|
||||
remote_kv_stride=stride[1],
|
||||
transfers_per_block=2,
|
||||
regions_per_block=2,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
elif shape[3] == spec.block_size:
|
||||
num_kv_heads, block_size, head_dim = shape[2:]
|
||||
else:
|
||||
kernel_num_blocks = num_blocks
|
||||
kernel_block_size, _, _ = _select_kernel_block_layout(
|
||||
layer_name, shape, spec
|
||||
)
|
||||
kernel_blocks_per_block = spec.block_size // kernel_block_size
|
||||
if kernel_num_blocks % kernel_blocks_per_block != 0:
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: "
|
||||
f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, "
|
||||
f"not divisible by {kernel_blocks_per_block}"
|
||||
)
|
||||
num_blocks = kernel_num_blocks // kernel_blocks_per_block
|
||||
block_size = spec.block_size
|
||||
block_stride = stride[0] * kernel_blocks_per_block
|
||||
block_len = block_stride * element_size
|
||||
slot_size_bytes = block_len // block_size
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=block_stride,
|
||||
local_kv_stride=None,
|
||||
remote_kv_stride=None,
|
||||
transfers_per_block=1,
|
||||
regions_per_block=1,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
slot_size_bytes = num_kv_heads * head_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[0],
|
||||
local_kv_stride=stride[1],
|
||||
remote_kv_stride=stride[1],
|
||||
transfers_per_block=2,
|
||||
regions_per_block=2,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
|
||||
if (
|
||||
isinstance(spec, AttentionSpec)
|
||||
and len(shape) == 4
|
||||
and shape[1] == spec.num_heads
|
||||
and shape[2] == spec.num_states
|
||||
and shape[3] * element_size == spec.state_content_size_bytes
|
||||
):
|
||||
# Standardized per-layer [B, H, N, C] view (MLA is just H == 1).
|
||||
num_blocks, num_heads, num_states, content_dim = shape
|
||||
slot_size_bytes = num_heads * content_dim * element_size
|
||||
block_len = num_states * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=spec.block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[0],
|
||||
local_kv_stride=None,
|
||||
remote_kv_stride=None,
|
||||
transfers_per_block=1,
|
||||
regions_per_block=1,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
|
||||
cache_kind = "MLA" if is_mla_cache else "K/V"
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO {cache_kind} cache shape for layer "
|
||||
f"{layer_name}: {tuple(shape)}"
|
||||
)
|
||||
|
||||
|
||||
def kda_conv_ssm(
|
||||
kv_cache: torch.Tensor,
|
||||
spec: MambaSpec,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Return zero-copy conv and SSM views over a packed two-state page."""
|
||||
from math import prod
|
||||
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
|
||||
if len(spec.shapes) != 2 or len(spec.dtypes) != 2:
|
||||
raise ValueError(
|
||||
"MoRIIO KDA transfer requires exactly two states (conv and SSM)"
|
||||
)
|
||||
if kv_cache.element_size() != 1:
|
||||
raise ValueError("MoRIIO KDA packed cache must use a byte-sized dtype")
|
||||
if kv_cache.ndim != 4 or kv_cache.shape[1:3] != (1, 1) or kv_cache.stride(3) != 1:
|
||||
raise ValueError(
|
||||
"MoRIIO KDA packed cache must have shape [B, 1, 1, C] with "
|
||||
"a contiguous byte dimension"
|
||||
)
|
||||
# Mirror MambaBase.bind_kv_cache: blocks may be strided within a shared
|
||||
# multi-layer allocation, while every individual byte page is contiguous.
|
||||
pages = kv_cache.squeeze(dim=(1, 2))
|
||||
states: list[torch.Tensor] = []
|
||||
offset = 0
|
||||
for shp, dt in zip(spec.shapes, spec.dtypes, strict=True):
|
||||
nbytes = prod(shp) * get_dtype_size(dt)
|
||||
if offset + nbytes > pages.shape[1]:
|
||||
raise ValueError(
|
||||
"MoRIIO KDA state layout exceeds the packed cache page size"
|
||||
)
|
||||
state = pages[:, offset : offset + nbytes].view(dt)
|
||||
states.append(state.view(-1, *shp))
|
||||
offset += nbytes
|
||||
return states[0], states[1]
|
||||
|
||||
|
||||
def iter_layer_registration_regions(
|
||||
layer_name: str,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_to_spec: Mapping[str, KVCacheSpec],
|
||||
) -> list[tuple[torch.Tensor, int]]:
|
||||
spec = layer_to_spec[layer_name]
|
||||
if isinstance(spec, MambaSpec):
|
||||
# KDA layer: register the conv and ssm tensors as two whole-tensor
|
||||
# regions (the conv 3-subprojection split is expressed as transfer
|
||||
# offset triples, not extra registrations).
|
||||
conv, ssm = kda_conv_ssm(kv_cache, spec)
|
||||
geom = get_mamba_transfer_geometry(conv, ssm)
|
||||
return [(conv, geom.conv_region_len), (ssm, geom.ssm_region_len)]
|
||||
geometry = cast(
|
||||
LayerTransferGeometry,
|
||||
get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec),
|
||||
)
|
||||
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
|
||||
if geometry.regions_per_block == 1:
|
||||
# Padded/interleaved pages stride past block_len: register the strided
|
||||
# span, ending at the last block's bytes so it stays in-allocation.
|
||||
block_stride_bytes = geometry.block_stride * kv_cache.element_size()
|
||||
region_len = max(
|
||||
region_len,
|
||||
(geometry.num_blocks - 1) * block_stride_bytes + geometry.block_len,
|
||||
)
|
||||
if geometry.split_kv_regions:
|
||||
return [(cache, region_len) for cache in kv_cache]
|
||||
return [(kv_cache, region_len)]
|
||||
|
||||
|
||||
def merge_contiguous_offsets(
|
||||
offsets_local: list[int],
|
||||
offsets_remote: list[int],
|
||||
sizes: list[int],
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
if not offsets_local:
|
||||
return [], [], []
|
||||
if not (len(offsets_local) == len(offsets_remote) == len(sizes)):
|
||||
raise ValueError("Input list lengths mismatch")
|
||||
|
||||
rows = sorted(zip(offsets_local, offsets_remote, sizes), key=lambda row: row[0])
|
||||
merged: list[list[int]] = []
|
||||
for local, remote, size in rows:
|
||||
if (
|
||||
merged
|
||||
and local == merged[-1][0] + merged[-1][2]
|
||||
and remote == merged[-1][1] + merged[-1][2]
|
||||
):
|
||||
merged[-1][2] += size
|
||||
else:
|
||||
merged.append([local, remote, size])
|
||||
|
||||
return (
|
||||
[row[0] for row in merged],
|
||||
[row[1] for row in merged],
|
||||
[row[2] for row in merged],
|
||||
)
|
||||
|
||||
|
||||
def compute_block_transfer_offsets(
|
||||
layer_name: str,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_to_spec: Mapping[str, KVCacheSpec],
|
||||
local_block_ids: list[int],
|
||||
remote_block_ids: list[int],
|
||||
remote_num_blocks: int,
|
||||
merge_fn: Callable[
|
||||
[list[int], list[int], list[int]], tuple[list[int], list[int], list[int]]
|
||||
] = merge_contiguous_offsets,
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
# A shorter/empty local list transfers only what decode allocated (READ-mode
|
||||
# partial or full-prefix-hit); a longer local list is a genuine bug.
|
||||
if len(local_block_ids) > len(remote_block_ids):
|
||||
raise ValueError(
|
||||
"local_block_ids longer than remote_block_ids: "
|
||||
f"{len(local_block_ids)} > {len(remote_block_ids)}"
|
||||
)
|
||||
geometry = cast(
|
||||
LayerTransferGeometry,
|
||||
get_layer_transfer_geometry(
|
||||
layer_name, kv_cache, layer_to_spec, remote_num_blocks
|
||||
),
|
||||
)
|
||||
invalid_local = next(
|
||||
(
|
||||
block_id
|
||||
for block_id in local_block_ids
|
||||
if not 0 <= block_id < geometry.num_blocks
|
||||
),
|
||||
None,
|
||||
)
|
||||
if invalid_local is not None:
|
||||
raise ValueError(
|
||||
f"local block id {invalid_local} is outside [0, {geometry.num_blocks})"
|
||||
)
|
||||
invalid_remote = next(
|
||||
(
|
||||
block_id
|
||||
for block_id in remote_block_ids
|
||||
if not 0 <= block_id < remote_num_blocks
|
||||
),
|
||||
None,
|
||||
)
|
||||
if invalid_remote is not None:
|
||||
raise ValueError(
|
||||
f"remote block id {invalid_remote} is outside [0, {remote_num_blocks})"
|
||||
)
|
||||
element_size = kv_cache.element_size()
|
||||
transfer_size_byte = geometry.block_len
|
||||
per_block = geometry.transfers_per_block
|
||||
total = len(local_block_ids) * per_block
|
||||
offset_local = [0] * total
|
||||
offset_remote = [0] * total
|
||||
sizes = [transfer_size_byte] * total
|
||||
|
||||
w = 0
|
||||
for lb, rb in zip(local_block_ids, remote_block_ids):
|
||||
offset_local[w] = element_size * (lb * geometry.block_stride)
|
||||
offset_remote[w] = element_size * (rb * geometry.block_stride)
|
||||
w += 1
|
||||
if per_block == 2:
|
||||
assert geometry.local_kv_stride is not None
|
||||
assert geometry.remote_kv_stride is not None
|
||||
offset_local[w] = element_size * (
|
||||
geometry.local_kv_stride + lb * geometry.block_stride
|
||||
)
|
||||
offset_remote[w] = element_size * (
|
||||
geometry.remote_kv_stride + rb * geometry.block_stride
|
||||
)
|
||||
w += 1
|
||||
|
||||
return merge_fn(offset_local, offset_remote, sizes)
|
||||
|
||||
|
||||
class MambaOffsetTemplate(NamedTuple):
|
||||
"""Slot-independent conv+ssm offset decomposition for a KDA layer.
|
||||
|
||||
Homogeneous-TP geometry (conv/ssm slot strides, conv sub-projection
|
||||
offsets, ssm per-slot size) is identical across requests and across the
|
||||
homogeneous GDN layers, so it is computed once and reused; only the
|
||||
per-request slot bases vary (see ``apply_mamba_offset_template``).
|
||||
"""
|
||||
|
||||
num_slots: int
|
||||
conv_slot_stride_bytes: int
|
||||
ssm_slot_stride_bytes: int
|
||||
ssm_read_bytes: int
|
||||
conv_subprojs: tuple[tuple[int, int, int], ...]
|
||||
|
||||
|
||||
def build_mamba_offset_template(
|
||||
conv: torch.Tensor,
|
||||
ssm: torch.Tensor,
|
||||
split_info: MambaConvSplitInfo,
|
||||
tp_ratio: int,
|
||||
) -> MambaOffsetTemplate:
|
||||
"""Precompute the slot-independent conv+ssm offset decomposition.
|
||||
|
||||
Only homogeneous TP (``tp_ratio == 1``) is supported; heterogeneous TP is
|
||||
gated with ``NotImplementedError`` (see the design doc). The returned
|
||||
template is a pure function of the layer geometry and ``split_info``, so it
|
||||
can be cached and reused across requests without recomputation.
|
||||
"""
|
||||
if tp_ratio != 1:
|
||||
# Heterogeneous-TP recurrent-state transfer needs the remote page's
|
||||
# slot stride (which differs from the local page) and, for
|
||||
# P_TP > D_TP, a multi-rank gather of conv/ssm slices. Both are
|
||||
# follow-ups. The mamba path bypasses the attention KV-head validator,
|
||||
# so fail loudly here instead of silently transferring wrong bytes.
|
||||
raise NotImplementedError(
|
||||
"heterogeneous-TP mamba/KDA transfer is not supported yet "
|
||||
f"(tp_ratio={tp_ratio}); use equal prefill/decode TP for "
|
||||
"hybrid models"
|
||||
)
|
||||
geom = get_mamba_transfer_geometry(conv, ssm)
|
||||
conv_local_offsets = split_info.local_conv_offsets
|
||||
conv_remote_offsets = split_info.remote_conv_offsets(0, 1)
|
||||
ssm_read_bytes = geom.ssm_slot_bytes
|
||||
conv_subprojs = tuple(
|
||||
(loff, roff, rsz)
|
||||
for (loff, _lsz), (roff, rsz) in zip(conv_local_offsets, conv_remote_offsets)
|
||||
)
|
||||
return MambaOffsetTemplate(
|
||||
num_slots=conv.shape[0],
|
||||
conv_slot_stride_bytes=geom.conv_slot_stride * geom.conv_element_size,
|
||||
ssm_slot_stride_bytes=geom.ssm_slot_stride * geom.ssm_element_size,
|
||||
ssm_read_bytes=ssm_read_bytes,
|
||||
conv_subprojs=conv_subprojs,
|
||||
)
|
||||
|
||||
|
||||
def apply_mamba_offset_template(
|
||||
template: MambaOffsetTemplate,
|
||||
local_slots: list[int],
|
||||
remote_slots: list[int],
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
"""Apply per-request slot bases to a cached ``MambaOffsetTemplate``.
|
||||
|
||||
Conv sub-projection entries (all slots) come first, followed by one ssm
|
||||
entry per slot.
|
||||
"""
|
||||
if len(local_slots) != len(remote_slots):
|
||||
raise ValueError(
|
||||
"local and remote mamba slot counts must match: "
|
||||
f"{len(local_slots)} != {len(remote_slots)}"
|
||||
)
|
||||
invalid_local = next(
|
||||
(slot for slot in local_slots if not 0 <= slot < template.num_slots), None
|
||||
)
|
||||
if invalid_local is not None:
|
||||
raise ValueError(
|
||||
f"local mamba slot {invalid_local} is outside [0, {template.num_slots})"
|
||||
)
|
||||
invalid_remote = next((slot for slot in remote_slots if slot < 0), None)
|
||||
if invalid_remote is not None:
|
||||
raise ValueError(f"remote mamba slot {invalid_remote} must be non-negative")
|
||||
# The peer's KDA slot count is not part of MoRIIOAgentMetadata yet. Do not
|
||||
# compare remote slot IDs against the local pool size: two otherwise
|
||||
# compatible engines may reserve different numbers of slots.
|
||||
local_offs: list[int] = []
|
||||
remote_offs: list[int] = []
|
||||
sizes: list[int] = []
|
||||
|
||||
# Conv sub-projections (region 0), all slots first.
|
||||
conv_slot_stride_bytes = template.conv_slot_stride_bytes
|
||||
for ls, rs in zip(local_slots, remote_slots):
|
||||
lbase = ls * conv_slot_stride_bytes
|
||||
rbase = rs * conv_slot_stride_bytes
|
||||
for loff, roff, rsz in template.conv_subprojs:
|
||||
local_offs.append(lbase + loff)
|
||||
remote_offs.append(rbase + roff)
|
||||
sizes.append(rsz)
|
||||
|
||||
# SSM temporal state (region 1), one entry per slot.
|
||||
ssm_slot_stride_bytes = template.ssm_slot_stride_bytes
|
||||
ssm_read_bytes = template.ssm_read_bytes
|
||||
for ls, rs in zip(local_slots, remote_slots):
|
||||
local_offs.append(ls * ssm_slot_stride_bytes)
|
||||
remote_offs.append(rs * ssm_slot_stride_bytes)
|
||||
sizes.append(ssm_read_bytes)
|
||||
|
||||
return local_offs, remote_offs, sizes
|
||||
|
||||
|
||||
def compute_mamba_conv_split_count(
|
||||
local_slots: list[int],
|
||||
split_info: MambaConvSplitInfo,
|
||||
) -> int:
|
||||
"""Number of leading conv entries in a KDA layer transfer plan.
|
||||
|
||||
Entries ``[:count]`` are conv sub-projections (conv region/session);
|
||||
entries ``[count:]`` are the per-slot ssm state (ssm region/session).
|
||||
"""
|
||||
return len(local_slots) * len(split_info.local_conv_offsets)
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NIXL KV-cache transfer connector (disaggregated prefill / decode)."""
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
|
||||
NixlBaseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
|
||||
NixlBaseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector import (
|
||||
NixlBaseConnector,
|
||||
NixlConnector,
|
||||
NixlPullConnector,
|
||||
NixlPushConnector,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlAgentMetadata,
|
||||
NixlConnectorMetadata,
|
||||
NixlHandshakePayload,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
|
||||
NixlPullConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
|
||||
NixlPushConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
|
||||
NixlConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
NixlKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"NixlAgentMetadata",
|
||||
"NixlBaseConnector",
|
||||
"NixlBaseConnectorScheduler",
|
||||
"NixlBaseConnectorWorker",
|
||||
"NixlConnector",
|
||||
"NixlConnectorMetadata",
|
||||
"NixlConnectorScheduler",
|
||||
"NixlConnectorWorker",
|
||||
"NixlHandshakePayload",
|
||||
"NixlKVConnectorStats",
|
||||
"NixlPullConnector",
|
||||
"NixlPullConnectorScheduler",
|
||||
"NixlPullConnectorWorker",
|
||||
"NixlPushConnector",
|
||||
"NixlPushConnectorScheduler",
|
||||
"NixlPushConnectorWorker",
|
||||
]
|
||||
@@ -1,549 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Base scheduler-side logic for the NIXL connector."""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import msgspec
|
||||
import zmq
|
||||
|
||||
from vllm import envs
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
EngineId,
|
||||
clip_ssm_state_blocks,
|
||||
yield_req_data,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorHandshakeMetadata,
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
GET_META_MSG,
|
||||
HeartbeatInfo,
|
||||
NixlConnectorMetadata,
|
||||
NixlHandshakePayload,
|
||||
ReqId,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import zmq_ctx
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NixlBaseConnectorScheduler:
|
||||
"""Base implementation of Scheduler side methods shared by pull and push."""
|
||||
|
||||
# Emitted in kv_transfer_params so an external router can distinguish a
|
||||
# pull (READ) producer from a push (WRITE) one. Overridden by the push
|
||||
# scheduler.
|
||||
_TRANSFER_MODE: str = "pull"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
engine_id: str,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# TP1 PCP+DCP exposes its DCP shards as transfer ranks.
|
||||
self.transfer_tp_size = max(
|
||||
parallel_config.tensor_parallel_size,
|
||||
parallel_config.decode_context_parallel_size,
|
||||
)
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.engine_id: EngineId = engine_id
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.side_channel_host = envs.VLLM_NIXL_SIDE_CHANNEL_HOST
|
||||
self.side_channel_port = (
|
||||
envs.VLLM_NIXL_SIDE_CHANNEL_PORT
|
||||
+ vllm_config.parallel_config.data_parallel_index
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self._kv_lease_duration: int = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"kv_lease_duration", 30
|
||||
)
|
||||
)
|
||||
# NOTE (NickLucche): For now we use a hardcoded value for a simpler interface.
|
||||
self._heartbeat_interval = self._kv_lease_duration // 6
|
||||
if current_platform.device_type == "cpu":
|
||||
self.use_host_buffer = False
|
||||
else:
|
||||
self.use_host_buffer = (
|
||||
vllm_config.kv_transfer_config.kv_buffer_device == "cpu"
|
||||
)
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
# Also handle unlikely SW-only model case instead of checking num_groups>1.
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
self._has_mamba = kv_cache_config.has_mamba_layers
|
||||
self._bounded_replay = any(
|
||||
g.kv_cache_spec.prefix_replay_tokens > 0
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
|
||||
logger.info("Initializing NIXL Scheduler %s", engine_id)
|
||||
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
|
||||
logger.info("Hybrid Memory Allocator is enabled with NIXL")
|
||||
|
||||
# Background thread for handling new handshake requests.
|
||||
self._nixl_handshake_listener_t: threading.Thread | None = None
|
||||
self._stop_event = threading.Event()
|
||||
|
||||
# Requests that need to start recv/send.
|
||||
# New requests are added by update_state_after_alloc in
|
||||
# the scheduler. Used to make metadata passed to Worker.
|
||||
self._reqs_need_recv: dict[
|
||||
ReqId, tuple[Request, BlockIds, tuple[int, ...], bool]
|
||||
] = {}
|
||||
self._reqs_need_save: dict[ReqId, Request] = {}
|
||||
# Reqs to send and their expiration time
|
||||
self._reqs_need_send: dict[ReqId, float] = {}
|
||||
self._reqs_in_batch: set[ReqId] = set()
|
||||
# Reqs to remove from processed set because they're not to send after
|
||||
# remote prefill or aborted.
|
||||
self._reqs_not_processed: set[ReqId] = set()
|
||||
|
||||
# Heartbeat tracking: requests needing periodic lease-renewal heartbeats to
|
||||
# remote P-side, stored as ready-to-send HeartbeatInfo grouped by remote engine
|
||||
self._heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
|
||||
# Reverse lookup: local req_id -> (engine_id, remote_req_id) for O(1) removal
|
||||
self._heartbeat_req_engine: dict[ReqId, tuple[EngineId, ReqId]] = {}
|
||||
self._last_heartbeat_time: float = 0.0
|
||||
|
||||
# Gather Sliding Window sizes for each kv cache group (if any) in number of
|
||||
# blocks per KV cache group. This is used to clip the local attention window.
|
||||
sw_sizes_tokens: list[tuple[int, int]] = [
|
||||
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
|
||||
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
|
||||
else (0, self.block_size)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
# cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively
|
||||
# account for boundary overlap eg window isn't fully aligned with blocks.
|
||||
self.blocks_per_sw = [
|
||||
cdiv(n_tokens, block_size) + 1 if n_tokens else 0
|
||||
for n_tokens, block_size in sw_sizes_tokens
|
||||
]
|
||||
|
||||
# Trailing scratch slots that mamba managers co-allocate per request
|
||||
# for speculative decoding; None for non-SSM groups.
|
||||
self._ssm_spec_blocks = [
|
||||
g.kv_cache_spec.num_speculative_blocks
|
||||
if isinstance(g.kv_cache_spec, MambaSpec)
|
||||
else None
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
|
||||
# Threshold to decide whether to compute kv cache locally
|
||||
# or pull from a remote node: minimum number of remote
|
||||
# tokens to amortize the xfer latencies
|
||||
self.kv_recompute_threshold: int = int(
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"kv_recompute_threshold", 64
|
||||
)
|
||||
)
|
||||
|
||||
# Bi-directional KV transfer feature supports KV block
|
||||
# transfers from D node to P node
|
||||
self.is_bidirectional_kv_xfer_enabled = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"bidirectional_kv_xfer", False
|
||||
)
|
||||
)
|
||||
self.decoder_kv_blocks_ttl = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"decoder_kv_blocks_ttl", 480
|
||||
)
|
||||
)
|
||||
|
||||
if self.is_bidirectional_kv_xfer_enabled and self.kv_recompute_threshold > 0:
|
||||
logger.info(
|
||||
"Bidirectional KV transfer is enabled and the kv "
|
||||
"recompute threshold is set to %d tokens."
|
||||
"KV blocks on D are released after a TTL of %d seconds.",
|
||||
self.kv_recompute_threshold,
|
||||
self.decoder_kv_blocks_ttl,
|
||||
)
|
||||
|
||||
def shutdown(self):
|
||||
self._stop_event.set()
|
||||
if self._nixl_handshake_listener_t is not None:
|
||||
self._nixl_handshake_listener_t.join()
|
||||
self._nixl_handshake_listener_t = None
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
"""Track a request that may need heartbeats."""
|
||||
params = request.kv_transfer_params
|
||||
if params is not None and params.get("do_remote_decode"):
|
||||
self._truncate_request_for_prefill(request)
|
||||
|
||||
# NOTE (NickLucche) This excludes request meant for P, ie heartbeats are
|
||||
# effectively disabled for Bidirectional KV transfer.
|
||||
if params is None or not params.get("do_remote_prefill"):
|
||||
return
|
||||
# Only track if all required remote fields are present.
|
||||
remote_engine_id = params.get("remote_engine_id")
|
||||
remote_request_id = params.get("remote_request_id")
|
||||
host = params.get("remote_host")
|
||||
port = params.get("remote_port")
|
||||
tp_size = params.get("tp_size")
|
||||
dcp_size = params.get("dcp_size", 1)
|
||||
pp_size = params.get("pp_size", 1)
|
||||
if (
|
||||
remote_engine_id is None
|
||||
or remote_request_id is None
|
||||
or host is None
|
||||
or port is None
|
||||
or tp_size is None
|
||||
):
|
||||
return
|
||||
if remote_engine_id not in self._heartbeat_by_engine:
|
||||
self._heartbeat_by_engine[remote_engine_id] = HeartbeatInfo(
|
||||
req_ids=set(),
|
||||
host=host,
|
||||
port=port,
|
||||
tp_size=tp_size,
|
||||
dcp_size=dcp_size,
|
||||
pp_size=pp_size,
|
||||
)
|
||||
self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id)
|
||||
self._heartbeat_req_engine[request.request_id] = (
|
||||
remote_engine_id,
|
||||
remote_request_id,
|
||||
)
|
||||
|
||||
def _stop_heartbeat(self, req_id: ReqId) -> None:
|
||||
"""Remove *req_id* from heartbeat tracking (if tracked)."""
|
||||
if key := self._heartbeat_req_engine.pop(req_id, None):
|
||||
engine_id, remote_id = key
|
||||
if info := self._heartbeat_by_engine.get(engine_id):
|
||||
info.req_ids.discard(remote_id)
|
||||
if not info.req_ids:
|
||||
# Clean up empty engines so we don't leak a key when remote dies.
|
||||
del self._heartbeat_by_engine[engine_id]
|
||||
|
||||
def get_exchange_clipped_blocks(
|
||||
self, block_ids: BlockIds, clip_ssm: bool = True
|
||||
) -> BlockIds:
|
||||
"""Clip a request's block lists down to the transferable blocks.
|
||||
|
||||
Sliding-window groups keep only the in-window tail: the KV cache
|
||||
manager allocates blocks for the entire sequence length and cleans up
|
||||
out-of-window blocks only prior to the `request_finished_all_groups`
|
||||
hook.
|
||||
|
||||
SSM groups keep only their state-bearing slot: the trailing
|
||||
speculative scratch slots and everything before the running state
|
||||
(null placeholders and the previous step's superseded state) go.
|
||||
|
||||
Use this at every block-id exchange point. Pass ``clip_ssm=False``
|
||||
for per-step partial lists (host-buffer save), where the SSM strip
|
||||
does not apply.
|
||||
"""
|
||||
if len(block_ids) == 0:
|
||||
# No blocks to clip, e.g. a full prefix cache hit.
|
||||
return block_ids
|
||||
block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
|
||||
if not self._is_hma_required:
|
||||
return block_ids
|
||||
# NOTE (NickLucche) This logic is currently handled at the connector level
|
||||
# because offloading connectors might want to receive the whole sequence even
|
||||
# for SWA groups. We will abstract this logic once the interface is more stable
|
||||
assert len(block_ids) == len(self.blocks_per_sw), (
|
||||
"Number of KV cache groups must match"
|
||||
)
|
||||
clipped = []
|
||||
for i, blocks in enumerate(block_ids):
|
||||
if n_sw := self.blocks_per_sw[i]:
|
||||
blocks = blocks[-n_sw:]
|
||||
elif (
|
||||
clip_ssm
|
||||
and blocks
|
||||
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
|
||||
):
|
||||
blocks = clip_ssm_state_blocks(blocks, n_spec_blocks)
|
||||
clipped.append(blocks)
|
||||
return tuple(clipped)
|
||||
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""Set the KV connector handshake metadata for this connector.
|
||||
|
||||
Args:
|
||||
metadata (dict): the handshake metadata to set.
|
||||
|
||||
"""
|
||||
encoded_data: dict[tuple[int, int], bytes] = {}
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
for (pp_rank, tp_rank), rank_metadata in metadata.items():
|
||||
if not isinstance(rank_metadata, NixlHandshakePayload):
|
||||
raise ValueError(
|
||||
"NixlConnectorScheduler expects NixlHandshakePayload for "
|
||||
"handshake metadata."
|
||||
)
|
||||
encoded_data[(pp_rank, tp_rank)] = encoder.encode(rank_metadata)
|
||||
logger.debug(
|
||||
"PP rank %d, TP rank %d: encoded NixlHandshakePayload size: %s bytes",
|
||||
pp_rank,
|
||||
tp_rank,
|
||||
str(len(encoded_data[(pp_rank, tp_rank)])),
|
||||
)
|
||||
|
||||
# Only start the listener when we have metadata to serve.
|
||||
if self._nixl_handshake_listener_t is None:
|
||||
ready_event = threading.Event()
|
||||
self._nixl_handshake_listener_t = threading.Thread(
|
||||
target=self._nixl_handshake_listener,
|
||||
args=(
|
||||
encoded_data,
|
||||
ready_event,
|
||||
self._stop_event,
|
||||
self.side_channel_host,
|
||||
self.side_channel_port,
|
||||
),
|
||||
daemon=True,
|
||||
name="nixl_handshake_listener",
|
||||
)
|
||||
self._nixl_handshake_listener_t.start()
|
||||
ready_event.wait() # Wait for listener ZMQ socket to be ready.
|
||||
|
||||
@staticmethod
|
||||
def _nixl_handshake_listener(
|
||||
encoded_data: dict[tuple[int, int], Any],
|
||||
ready_event: threading.Event,
|
||||
stop_event: threading.Event,
|
||||
host: str,
|
||||
port: int,
|
||||
):
|
||||
"""Background thread for getting new NIXL handshakes."""
|
||||
# NOTE(rob): this is a simple implementation. We will move
|
||||
# to a better approach via HTTP endpoint soon.
|
||||
|
||||
# Listen for new requests for metadata.
|
||||
path = make_zmq_path("tcp", host, port)
|
||||
logger.debug("Starting listening on path: %s", path)
|
||||
with zmq_ctx(zmq.ROUTER, path) as sock:
|
||||
sock.setsockopt(zmq.RCVTIMEO, 1000)
|
||||
ready_event.set()
|
||||
while True:
|
||||
try:
|
||||
identity, _, msg = sock.recv_multipart()
|
||||
except zmq.Again:
|
||||
if stop_event.is_set():
|
||||
break
|
||||
continue
|
||||
# Decode (GET_META_MSG, pp_rank, tp_rank).
|
||||
msg, target_pp_rank, target_tp_rank = msgspec.msgpack.decode(msg)
|
||||
logger.debug(
|
||||
"Received message for pp rank %s, tp rank %s",
|
||||
target_pp_rank,
|
||||
target_tp_rank,
|
||||
)
|
||||
if msg != GET_META_MSG:
|
||||
logger.warning("Connection listener got unexpected message %s", msg)
|
||||
# Echo our perf_counter so P can estimate the clock offset.
|
||||
# perf_counter is only comparable within a process, so this
|
||||
# listener must run in the same process that stamps the block
|
||||
# expiry deadline (`_reqs_need_send`).
|
||||
ts = msgspec.msgpack.encode(time.perf_counter())
|
||||
sock.send_multipart(
|
||||
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)], ts)
|
||||
)
|
||||
|
||||
def _prefill_backoff(self) -> int:
|
||||
"""Trailing prompt tokens the prefiller must not compute; the decoder
|
||||
recomputes them locally.
|
||||
|
||||
Mamba needs h(N-1) so the decoder can derive h(N) itself. DSV41's sliding
|
||||
window under bounded replay is not replayed after a P/D load, so it must
|
||||
be laid out for the token the decoder recomputes. Multi-module
|
||||
MTP needs to keep its whole lookahead window off of the prefiller, which
|
||||
would otherwise embed the unverified drafts in the MTP layer's KV cache. The
|
||||
decoder would never rebuild them, because the update is sized by the rejection
|
||||
count, which is zero for the first decode.
|
||||
"""
|
||||
return max(
|
||||
1 if self._has_mamba or self._bounded_replay else 0,
|
||||
self.vllm_config.num_prefill_lookahead_tokens - 1,
|
||||
)
|
||||
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. The number of prompt tokens to load from the prefiller.
|
||||
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
|
||||
will recompute locally."""
|
||||
backoff = self._prefill_backoff()
|
||||
if backoff and num_prompt_tokens > backoff:
|
||||
return num_prompt_tokens - backoff
|
||||
return num_prompt_tokens
|
||||
|
||||
def _truncate_request_for_prefill(self, request: "Request") -> None:
|
||||
"""P-side only: drop the trailing ``_prefill_backoff()`` prompt tokens
|
||||
so the prefiller stops short of what the decoder recomputes locally.
|
||||
For Mamba that is the single token needed to yield h(N-1); for
|
||||
multi-module MTP it is the drafter's whole lookahead window.
|
||||
|
||||
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
|
||||
request is preempted and rescheduled."""
|
||||
backoff = self._prefill_backoff()
|
||||
params = request.kv_transfer_params
|
||||
if (
|
||||
backoff
|
||||
and params is not None
|
||||
# Guard against repeated truncation after preemption/reschedule.
|
||||
and not params.get("_p_side_truncated")
|
||||
and request.num_prompt_tokens > backoff
|
||||
# A request that skips reading the prefix cache (prompt logprobs)
|
||||
# loads nothing on the decoder, which recomputes its whole prompt.
|
||||
and not request.get_skip_reading_prefix_cache()
|
||||
):
|
||||
if request.prompt_token_ids is not None:
|
||||
del request.prompt_token_ids[-backoff:]
|
||||
elif request.prompt_embeds is not None:
|
||||
request.prompt_embeds = request.prompt_embeds[:-backoff]
|
||||
else:
|
||||
return
|
||||
|
||||
del request._all_token_ids[-backoff:]
|
||||
request.num_prompt_tokens -= backoff
|
||||
request.max_tokens = 1
|
||||
params["_p_side_truncated"] = True
|
||||
|
||||
def _build_save_meta(
|
||||
self,
|
||||
meta: NixlConnectorMetadata,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> None:
|
||||
# only called when use_host_buffer is True to build the save metadata
|
||||
|
||||
# NOTE: For the prefill side, there might be a chance that an early added
|
||||
# request is a chunked prefill, so we need to check if new blocks are added
|
||||
for req_id, new_block_id_groups, _ in yield_req_data(scheduler_output):
|
||||
req_to_save = self._reqs_need_save.get(req_id)
|
||||
if req_to_save is None or new_block_id_groups is None:
|
||||
continue
|
||||
req = req_to_save
|
||||
|
||||
assert req.kv_transfer_params is not None
|
||||
clipped_block_id_groups = self.get_exchange_clipped_blocks(
|
||||
new_block_id_groups, clip_ssm=False
|
||||
)
|
||||
meta.add_new_req_to_save(
|
||||
request_id=req_id,
|
||||
local_block_ids=clipped_block_id_groups,
|
||||
kv_transfer_params=req.kv_transfer_params,
|
||||
)
|
||||
assert scheduler_output.num_scheduled_tokens is not None
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
is_partial = (
|
||||
req.num_computed_tokens + num_scheduled_tokens
|
||||
) < req.num_prompt_tokens
|
||||
if not is_partial:
|
||||
# For non-partial prefills, once new req_meta is scheduled, it
|
||||
# can be removed from _reqs_need_save.
|
||||
# For partial prefill case, we will retain the request in
|
||||
# _reqs_need_save until all blocks are scheduled with req_meta.
|
||||
# Therefore, only pop if `not is_partial`.
|
||||
self._reqs_need_save.pop(req_id)
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
meta = NixlConnectorMetadata()
|
||||
|
||||
# Loop through scheduled reqs and convert to ReqMeta.
|
||||
for req_id, (
|
||||
req,
|
||||
block_ids,
|
||||
cached,
|
||||
awaiting_kvs,
|
||||
) in self._reqs_need_recv.items():
|
||||
assert req.kv_transfer_params is not None
|
||||
meta.add_new_req_to_recv(
|
||||
request_id=req_id,
|
||||
local_block_ids=block_ids,
|
||||
kv_transfer_params=req.kv_transfer_params,
|
||||
local_num_computed_blocks=cached,
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
|
||||
if self.use_host_buffer:
|
||||
self._build_save_meta(meta, scheduler_output)
|
||||
|
||||
meta.reqs_to_send = self._reqs_need_send
|
||||
# Clock reference for reqs_to_send: deadlines above are in this
|
||||
# process's perf_counter domain; workers (possibly on other nodes,
|
||||
# where perf_counter has a different epoch) rebase against this.
|
||||
meta.scheduler_clock = time.perf_counter()
|
||||
meta.reqs_in_batch = self._reqs_in_batch
|
||||
meta.reqs_not_processed = self._reqs_not_processed
|
||||
|
||||
# Package heartbeats, throttled by heartbeat_interval.
|
||||
if self._heartbeat_by_engine:
|
||||
now = time.perf_counter()
|
||||
if now - self._last_heartbeat_time >= self._heartbeat_interval:
|
||||
self._last_heartbeat_time = now
|
||||
meta.heartbeat_by_engine = self._heartbeat_by_engine
|
||||
|
||||
# Clear the list once workers start the transfers
|
||||
self._reqs_need_recv.clear()
|
||||
self._reqs_in_batch = set()
|
||||
self._reqs_not_processed = set()
|
||||
self._reqs_need_send = {}
|
||||
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: "KVConnectorOutput") -> None:
|
||||
"""Stop heartbeating for requests whose KV transfer completed."""
|
||||
for req_id in connector_output.finished_recving or ():
|
||||
self._stop_heartbeat(req_id)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
return False
|
||||
|
||||
############################################################
|
||||
# Abstract methods that subclasses must implement
|
||||
############################################################
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
raise NotImplementedError
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: BlockIds,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
raise NotImplementedError
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,426 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NIXL connector facades.
|
||||
|
||||
This module hosts the thin facade classes that vLLM's KV-connector layer
|
||||
instantiates. Almost all the real work lives in the per-mode scheduler
|
||||
and worker classes; the connector classes here only forward calls.
|
||||
|
||||
* :class:`NixlBaseConnector` – common logic shared by pull and push.
|
||||
* :class:`NixlPullConnector` – pull-based (READ) KV transfer.
|
||||
* :class:`NixlPushConnector` – push-based (WRITE) KV transfer.
|
||||
* ``NixlConnector`` – backward-compatible alias for :class:`NixlPullConnector`.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineId,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
CopyBlocksOp,
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorHandshakeMetadata,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
|
||||
NixlPullConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
|
||||
NixlPushConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
NixlKVConnectorStats,
|
||||
NixlPromMetrics,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
|
||||
NixlBaseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
|
||||
NixlBaseConnectorWorker,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""Base connector with common logic shared by pull and push modes."""
|
||||
|
||||
_cache_hit_source = CacheHitSource.P2P
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
return True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
assert vllm_config.kv_transfer_config.engine_id is not None
|
||||
|
||||
if vllm_config.kv_transfer_config.kv_role == "kv_both":
|
||||
logger.warning_once(
|
||||
"Using kv_role='kv_both' with NixlConnector is deprecated "
|
||||
"and will be removed in a future release. Please set "
|
||||
"kv_role='kv_producer' for prefill instances and "
|
||||
"kv_role='kv_consumer' for decode instances. "
|
||||
)
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
pcp_size = parallel_config.prefill_context_parallel_size
|
||||
kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
if pcp_size > 1 and kv_role in ("kv_consumer", "kv_both"):
|
||||
raise NotImplementedError(
|
||||
"NixlConnector PCP currently supports kv_producer only. "
|
||||
"Consumers and kv_both require "
|
||||
"prefill_context_parallel_size=1."
|
||||
)
|
||||
dcp_size = parallel_config.decode_context_parallel_size
|
||||
if (
|
||||
pcp_size > 1
|
||||
and dcp_size > 1
|
||||
and (parallel_config.tensor_parallel_size != 1 or dcp_size != pcp_size)
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"NixlConnector PCP+DCP requires TP1 with DCP spanning "
|
||||
"the full PCP group."
|
||||
)
|
||||
# TODO: Support PCP with bidirectional KV transfer by tracking separate
|
||||
# send and receive completion counts.
|
||||
if pcp_size > 1 and vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"bidirectional_kv_xfer", False
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"NixlConnector PCP producers do not support bidirectional KV transfer."
|
||||
)
|
||||
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id
|
||||
self.kv_transfer_config = vllm_config.kv_transfer_config
|
||||
# Subclasses must set self.connector_scheduler and self.connector_worker
|
||||
self.connector_scheduler: NixlBaseConnectorScheduler | None = None
|
||||
self.connector_worker: NixlBaseConnectorWorker | None = None
|
||||
|
||||
############################################################
|
||||
# Class Methods
|
||||
############################################################
|
||||
@classmethod
|
||||
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: VllmConfig):
|
||||
if vllm_config.model_config is None:
|
||||
logger.warning_once(
|
||||
"Unable to detect current VLLM config. "
|
||||
"Fallback to default kv cache layout."
|
||||
)
|
||||
return None
|
||||
use_mla = vllm_config.model_config.use_mla
|
||||
if use_mla:
|
||||
# return None when we have mla
|
||||
# as the layout should not matter in that case,
|
||||
# which fallback to the default behavior.
|
||||
return None
|
||||
logger.info_once(
|
||||
"NixlConnector setting KV cache layout to LBHNC for "
|
||||
"better xfer performance."
|
||||
)
|
||||
return "LBHNC"
|
||||
|
||||
############################################################
|
||||
# Scheduler Side Methods
|
||||
############################################################
|
||||
|
||||
def get_finished_count(self) -> int | None:
|
||||
return None
|
||||
|
||||
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
|
||||
return self._kv_cache_config.transfer_group_ids
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.update_state_after_alloc(
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.on_new_request(request)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, (block_ids,))
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, block_ids)
|
||||
|
||||
def set_xfer_handshake_metadata_pp_aware(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""Set handshake metadata keyed by (pp_rank, tp_rank) so the side
|
||||
channel can serve every PP stage's agent metadata.
|
||||
|
||||
Args:
|
||||
metadata (dict): the handshake metadata to set.
|
||||
|
||||
"""
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.set_xfer_handshake_metadata(metadata)
|
||||
|
||||
############################################################
|
||||
# Worker Side Methods
|
||||
############################################################
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.set_host_xfer_buffer_ops(copy_operation)
|
||||
|
||||
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
|
||||
"""Get the finished recving and sending requests."""
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished()
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
assert self.connector_worker is not None
|
||||
results = self.connector_worker.get_transfer_results()
|
||||
if (
|
||||
self.kv_transfer_config.kv_role == "kv_producer"
|
||||
and self.connector_worker.pcp_rank > 0
|
||||
and not self.connector_worker.pcp_dcp_sharded
|
||||
):
|
||||
results.finished_sending.clear()
|
||||
return results
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get block IDs that failed to load via NIXL."""
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
NixlKVConnectorStats(data=data)
|
||||
if data is not None
|
||||
else NixlKVConnectorStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return NixlPromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""NixlConnector does not do layerwise saving."""
|
||||
pass
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""NixlConnector does not save explicitly."""
|
||||
pass
|
||||
|
||||
def wait_for_save(self):
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
|
||||
if self.connector_worker.use_host_buffer and self.connector_worker.copy_blocks:
|
||||
self.connector_worker.save_kv_to_host(self._connector_metadata)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
if self.connector_scheduler is not None:
|
||||
return self.connector_scheduler.has_pending_push_work()
|
||||
return False
|
||||
|
||||
def shutdown(self):
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.shutdown()
|
||||
if self.connector_scheduler is not None:
|
||||
self.connector_scheduler.shutdown()
|
||||
|
||||
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
|
||||
"""Get the KVConnector handshake metadata for this connector.
|
||||
This metadata is used for out-of-band connector handshake
|
||||
between P/D workers.
|
||||
|
||||
Returns:
|
||||
KVConnectorHandshakeMetadata: the handshake metadata.
|
||||
None if no handshake metadata is available.
|
||||
|
||||
"""
|
||||
assert self.connector_worker is not None
|
||||
if (
|
||||
self.kv_transfer_config.kv_role == "kv_producer"
|
||||
and self.connector_worker.pcp_rank > 0
|
||||
and not self.connector_worker.pcp_dcp_sharded
|
||||
):
|
||||
return None
|
||||
return self.connector_worker.xfer_handshake_metadata
|
||||
|
||||
|
||||
class NixlPullConnector(NixlBaseConnector):
|
||||
"""Pull-based (READ) NIXL KV transfer connector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = NixlPullConnectorScheduler(
|
||||
vllm_config, self.engine_id, kv_cache_config
|
||||
)
|
||||
self.connector_worker = None
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_scheduler = None
|
||||
self.connector_worker = NixlPullConnectorWorker(
|
||||
vllm_config, self.engine_id, kv_cache_config
|
||||
)
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self.connector_worker, NixlPullConnectorWorker)
|
||||
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
|
||||
self.connector_worker.start_load_kv(self._connector_metadata)
|
||||
|
||||
|
||||
class NixlPushConnector(NixlBaseConnector):
|
||||
"""Push-based (WRITE) NIXL KV transfer connector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
if vllm_config.parallel_config.decode_context_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"NixlPushConnector does not support decode_context_parallel_size > 1."
|
||||
)
|
||||
self.connector_scheduler: NixlPushConnectorScheduler | None = None
|
||||
self.connector_worker: NixlPushConnectorWorker | None = None
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = NixlPushConnectorScheduler(
|
||||
vllm_config, self.engine_id, kv_cache_config
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_worker = NixlPushConnectorWorker(
|
||||
vllm_config, self.engine_id, kv_cache_config
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported KVConnectorRole: {role}")
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
"""Drive push processing on the worker.
|
||||
|
||||
The worker enqueues registrations / finished blocks for the
|
||||
background ``nixl-push-writer`` thread; the writer issues the
|
||||
WRITE transfers and polls NIXL notifs without further
|
||||
engine-thread involvement.
|
||||
"""
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self._connector_metadata, NixlConnectorMetadata)
|
||||
self.connector_worker.start_load_kv(self._connector_metadata)
|
||||
|
||||
|
||||
# Backward compatibility: NixlConnector is the pull-based connector.
|
||||
NixlConnector = NixlPullConnector
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NixlBaseConnector",
|
||||
"NixlConnector",
|
||||
"NixlPullConnector",
|
||||
"NixlPushConnector",
|
||||
]
|
||||
@@ -1,340 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Metadata dataclasses and helpers for the NIXL connector."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds, EngineId
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorHandshakeMetadata,
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
TransferHandle = int
|
||||
ReqId = str
|
||||
|
||||
GET_META_MSG = b"get_meta_msg"
|
||||
|
||||
# Push-mode (WRITE-based) registration notification.
|
||||
# Sent worker-to-worker over NIXL: D worker -> P worker, encoded as
|
||||
# PUSH_REG_NOTIF_PREFIX + msgpack(registration_data).
|
||||
PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
||||
#
|
||||
# NIXL Connector Version
|
||||
#
|
||||
# Increment this version whenever there is an incompatible change to:
|
||||
# - NixlAgentMetadata schema
|
||||
# - kv_transfer_params schema or semantics
|
||||
# - NIXL transfer protocol or wire format
|
||||
# - KV cache memory layout or block organization
|
||||
# - Any other change that breaks P/D interoperability
|
||||
#
|
||||
# Version History:
|
||||
# 1: Initial version with compatibility checking
|
||||
# 2: Add remote_request_id to kv_transfer_params
|
||||
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
|
||||
# 4: Add KV block lease renewal through heartbeats
|
||||
# 5: Add remote_blocks_expiry_time to kv_transfer_params + handshake
|
||||
# clock-sync timestamp
|
||||
# 6: Validate EAGLE/MTP speculative configuration compatibility
|
||||
# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash
|
||||
# 8: Add dcp_size and pcp_size to NixlAgentMetadata
|
||||
# 9: Add block_strides
|
||||
# 10: Add dense virtual transfer pages for compressed MLA caches
|
||||
# 11: Add per-region transfer geometry and memory types to NixlAgentMetadata
|
||||
# 12: Add per-region member names for PP push
|
||||
# 13: Add packed-member layouts and order-independent packed-push backend hashes
|
||||
# 14: Add Qwen3.8-Flash-Next PLE page length - #59997
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 14
|
||||
|
||||
|
||||
@dataclass
|
||||
class NixlAgentMetadata:
|
||||
engine_id: str
|
||||
agent_metadata: bytes
|
||||
kv_caches_base_addr: list[int]
|
||||
device_id: int
|
||||
num_blocks: int
|
||||
block_lens: list[int]
|
||||
block_strides: list[int]
|
||||
kv_cache_layout: str
|
||||
block_size: int
|
||||
ssm_sizes: tuple[int, int]
|
||||
attn_backend_name: str
|
||||
physical_blocks_per_logical_kv_block: int
|
||||
region_num_blocks: list[int] | None = None
|
||||
region_group_ids: list[int] | None = None
|
||||
region_names: list[str] | None = None
|
||||
region_mem_types: list[str] | None = None
|
||||
dcp_size: int = 1
|
||||
pcp_size: int = 1
|
||||
# Page length of the PLE short-conv state (None when the model has none).
|
||||
ple_block_len: int | None = None
|
||||
# Layer names sharing each advertised region, in region order.
|
||||
region_members: list[list[str]] = field(default_factory=list)
|
||||
# Packed member -> (byte offset in its region's block, bytes per page).
|
||||
packed_member_layouts: dict[str, tuple[int, int]] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NixlHandshakePayload(KVConnectorHandshakeMetadata):
|
||||
"""Wrapper for NIXL handshake sent over the wire.
|
||||
|
||||
Enables two-phase decoding for graceful compatibility checking:
|
||||
1. Decode NixlHandshakePayload to get compatibility_hash
|
||||
2. Compute local hash and compare
|
||||
3. Only if hashes match, decode agent_metadata_bytes
|
||||
|
||||
This prevents decoder errors when NixlAgentMetadata schema is
|
||||
incompatible, allowing graceful failure with clear error message.
|
||||
"""
|
||||
|
||||
compatibility_hash: str
|
||||
agent_metadata_bytes: bytes # NixlAgentMetadata encoded
|
||||
|
||||
|
||||
def _get_speculative_compatibility_factors(
|
||||
vllm_config: VllmConfig,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Return NIXL compatibility factors for hidden-state-based speculators."""
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None or not speculative_config.use_eagle():
|
||||
return None
|
||||
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
assert draft_model_config is not None
|
||||
auxiliary_layer_ids = getattr(
|
||||
draft_model_config.hf_config,
|
||||
"eagle_aux_hidden_state_layer_ids",
|
||||
None,
|
||||
)
|
||||
|
||||
# kv_cache_dtype is a user override that defaults to None, meaning "inherit
|
||||
# the target's --kv-cache-dtype". Resolve it to the effective value so an
|
||||
# explicit setting on one side and inheritance on the other (same effective
|
||||
# dtype) don't spuriously mismatch.
|
||||
kv_cache_dtype = (
|
||||
speculative_config.kv_cache_dtype or vllm_config.cache_config.cache_dtype
|
||||
)
|
||||
|
||||
# Note: the draft attention_backend is intentionally not hashed. Its only
|
||||
# transfer-relevant effect is the KV block layout/size, which is validated
|
||||
# per region at runtime in _validate_remote_agent_handshake. The connector
|
||||
# only sees the raw override here (usually None = auto-select), never the
|
||||
# resolved backend, so hashing it would cause false mismatches without
|
||||
# catching anything the runtime layout check misses.
|
||||
return {
|
||||
"method": speculative_config.method,
|
||||
"model": draft_model_config.model,
|
||||
"revision": draft_model_config.revision,
|
||||
"code_revision": draft_model_config.code_revision,
|
||||
"parallel_drafting": speculative_config.parallel_drafting,
|
||||
"kv_cache_dtype": str(kv_cache_dtype),
|
||||
"auxiliary_layer_ids": (
|
||||
tuple(auxiliary_layer_ids) if auxiliary_layer_ids is not None else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def compute_nixl_compatibility_hash(
|
||||
vllm_config: VllmConfig,
|
||||
attn_backend_name: str,
|
||||
transfer_mode: str = "pull",
|
||||
) -> str:
|
||||
"""Compute compatibility hash for NIXL KV transfer.
|
||||
|
||||
Hash only the factors that affect whether two NIXL instances can
|
||||
successfully transfer KV cache data.
|
||||
|
||||
Factors included:
|
||||
- vLLM version and NIXL connector version
|
||||
- Model architecture (name, dtype, KV heads, layers)
|
||||
- KV cache format (dtype, sliding window)
|
||||
- Attention backend
|
||||
- EAGLE/MTP configuration that affects transferred state
|
||||
- Transfer mode (push vs pull)
|
||||
|
||||
The transfer mode is included because the push (WRITE) and pull (READ)
|
||||
connectors use incompatible transfer protocols; a push connector and a
|
||||
pull connector must never complete a handshake with each other.
|
||||
|
||||
Note: Factors like tensor_parallel_size, block_size, and kv_cache_layout
|
||||
are validated at runtime in _validate_remote_agent_handshake and are not
|
||||
included in this hash to support heterogeneous deployments.
|
||||
|
||||
Note - the set of factors are likely to evolve significantly over
|
||||
time to be more or less permissive.
|
||||
|
||||
Returns:
|
||||
SHA-256 hex digest
|
||||
|
||||
"""
|
||||
from vllm import __version__ as vllm_version
|
||||
from vllm.config.utils import hash_factors
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
is_hma_enabled = not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
|
||||
factors = {
|
||||
# Version compatibility
|
||||
"vllm_version": vllm_version,
|
||||
"nixl_connector_version": NIXL_CONNECTOR_VERSION,
|
||||
# Model architecture - affects KV cache shape
|
||||
"model": model_config.model,
|
||||
"dtype": str(model_config.dtype),
|
||||
"num_kv_heads": model_config.get_total_num_kv_heads(),
|
||||
"head_size": model_config.get_head_size(),
|
||||
"num_hidden_layers": model_config.get_total_num_hidden_layers(),
|
||||
# Attention backend and KV cache dtype affect memory layout
|
||||
"attn_backend_name": attn_backend_name,
|
||||
"cache_dtype": str(cache_config.cache_dtype),
|
||||
"is_hma_enabled": is_hma_enabled,
|
||||
"speculative_config": _get_speculative_compatibility_factors(vllm_config),
|
||||
# push (WRITE) and pull (READ) connectors are protocol-incompatible
|
||||
"transfer_mode": transfer_mode,
|
||||
}
|
||||
|
||||
compat_hash = hash_factors(factors)
|
||||
logger.debug(
|
||||
"NIXL compatibility hash: %s (model=%s, dtype=%s, num_kv_heads=%d, "
|
||||
"cache_dtype=%s, attn_backend=%s)",
|
||||
compat_hash,
|
||||
factors["model"],
|
||||
factors["dtype"],
|
||||
factors["num_kv_heads"],
|
||||
factors["cache_dtype"],
|
||||
attn_backend_name,
|
||||
)
|
||||
return compat_hash
|
||||
|
||||
|
||||
@dataclass
|
||||
class HeartbeatInfo:
|
||||
"""Heartbeat data for a single remote engine, sent from D worker to P."""
|
||||
|
||||
req_ids: set[ReqId]
|
||||
host: str
|
||||
port: int
|
||||
tp_size: int
|
||||
dcp_size: int = 1
|
||||
pp_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class RemoteMeta:
|
||||
block_ids: BlockIds
|
||||
host: str
|
||||
port: int
|
||||
engine_id: str
|
||||
request_id: str
|
||||
blocks_expiry_time: float | None = None
|
||||
num_tokens: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
local_block_ids: BlockIds
|
||||
# To be used when logical block size does not match the kernel block size
|
||||
local_physical_block_ids: BlockIds
|
||||
tp_size: int
|
||||
dcp_size: int = 1
|
||||
# Per-KV-cache-group logical blocks this rank already holds, i.e. its
|
||||
# prefix-cache hit. Fixes where this rank's DCP slice starts relative to
|
||||
# the remote's; kept per-group since hybrid models (e.g. SWA+FA) can have
|
||||
# different cache-hit counts per group.
|
||||
local_num_computed_blocks: tuple[int, ...] = ()
|
||||
remote: RemoteMeta | None = None
|
||||
# Remote block size, discovered during NIXL handshake (push mode).
|
||||
remote_block_size: int | None = None
|
||||
# Remote producer pipeline-parallel size (push mode, D side).
|
||||
pp_size: int = 1
|
||||
# True only when the scheduler parked the request in WAITING_FOR_REMOTE_KVS
|
||||
# and expects it in finished_recving; notify-only recvs must not be reported.
|
||||
awaiting_kvs: bool = False
|
||||
# Worker-only, per-region physical pages to zero after a successful pull.
|
||||
# None selects group-based completion; empty lists mean no zeroing.
|
||||
region_blocks_to_zero: BlockIds | None = None
|
||||
|
||||
|
||||
class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
def __init__(self):
|
||||
self.reqs_to_recv: dict[ReqId, ReqMeta] = {}
|
||||
self.reqs_to_save: dict[ReqId, ReqMeta] = {}
|
||||
self.reqs_to_send: dict[ReqId, float] = {}
|
||||
# The scheduler process's time.perf_counter() when this metadata was
|
||||
# built. reqs_to_send deadlines are stamped with the scheduler's
|
||||
# clock, which is NOT comparable across processes (perf_counter is
|
||||
# process/boot-local): workers must rebase the remaining TTL onto
|
||||
# their own clock via this reference. 0.0 = unset (legacy metadata).
|
||||
self.scheduler_clock: float = 0.0
|
||||
self.reqs_in_batch: set[ReqId] = set()
|
||||
self.reqs_not_processed: set[ReqId] = set()
|
||||
# Heartbeat data grouped by remote engine, sent by D worker to P.
|
||||
self.heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
|
||||
# Push mode (D side): registration data the D worker should send to
|
||||
# P workers via NIXL notification on this step.
|
||||
self.push_registrations: dict[ReqId, dict[str, Any]] = {}
|
||||
# Push mode (P side): newly finished request blocks to be matched
|
||||
# against pending D registrations on the P worker.
|
||||
self.push_finished_blocks: dict[ReqId, BlockIds] = {}
|
||||
|
||||
def _add_new_req(
|
||||
self,
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
local_num_computed_blocks: tuple[int, ...] = (),
|
||||
awaiting_kvs: bool = False,
|
||||
) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
local_block_ids=local_block_ids,
|
||||
local_physical_block_ids=local_block_ids,
|
||||
# P workers don't need to receive these from proxy here.
|
||||
tp_size=kv_transfer_params.get("tp_size", 1),
|
||||
dcp_size=kv_transfer_params.get("dcp_size", 1),
|
||||
remote_block_size=kv_transfer_params.get("remote_block_size"),
|
||||
pp_size=kv_transfer_params.get("pp_size", 1),
|
||||
local_num_computed_blocks=local_num_computed_blocks,
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
|
||||
def add_new_req_to_save(
|
||||
self,
|
||||
request_id: ReqId,
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
):
|
||||
self.reqs_to_save[request_id] = self._add_new_req(
|
||||
local_block_ids, kv_transfer_params
|
||||
)
|
||||
|
||||
def add_new_req_to_recv(
|
||||
self,
|
||||
request_id: ReqId,
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
local_num_computed_blocks: tuple[int, ...] = (),
|
||||
awaiting_kvs: bool = False,
|
||||
):
|
||||
req = self._add_new_req(
|
||||
local_block_ids,
|
||||
kv_transfer_params,
|
||||
local_num_computed_blocks,
|
||||
awaiting_kvs,
|
||||
)
|
||||
req.remote = RemoteMeta(
|
||||
block_ids=kv_transfer_params["remote_block_ids"],
|
||||
engine_id=kv_transfer_params["remote_engine_id"],
|
||||
request_id=kv_transfer_params["remote_request_id"],
|
||||
host=kv_transfer_params["remote_host"],
|
||||
port=kv_transfer_params["remote_port"],
|
||||
blocks_expiry_time=kv_transfer_params.get("remote_blocks_expiry_time"),
|
||||
num_tokens=kv_transfer_params.get("remote_num_tokens"),
|
||||
)
|
||||
self.reqs_to_recv[request_id] = req
|
||||
@@ -1,292 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pull-specific scheduler-side logic for the NIXL connector."""
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
|
||||
NixlBaseConnectorScheduler,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
"""Pull-specific scheduler logic (READ-based KV transfer)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
engine_id: str,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
"""For remote prefill, pull all prompt blocks from remote
|
||||
asynchronously relative to engine execution.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally
|
||||
computed tokens for this request
|
||||
Returns:
|
||||
* the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
* true if the external KV cache tokens will be loaded
|
||||
asynchronously (between scheduler steps).
|
||||
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NIXLConnector get_num_new_matched_tokens: "
|
||||
"num_computed_tokens=%s, kv_transfer_params=%s",
|
||||
num_computed_tokens,
|
||||
params,
|
||||
)
|
||||
|
||||
if params is not None and params.get("do_remote_prefill"):
|
||||
# Remote prefill: get all prompt blocks from remote.
|
||||
token_ids = request.prompt_token_ids or []
|
||||
actual = self._get_remote_prefill_token_count(len(token_ids))
|
||||
count = actual - num_computed_tokens
|
||||
if count > 0:
|
||||
return count, True
|
||||
|
||||
if (
|
||||
params is not None
|
||||
and params.get("do_remote_decode")
|
||||
and params.get("remote_block_ids")
|
||||
and all(
|
||||
p in params
|
||||
for p in (
|
||||
"remote_engine_id",
|
||||
"remote_request_id",
|
||||
"remote_host",
|
||||
"remote_port",
|
||||
)
|
||||
)
|
||||
):
|
||||
# Decode node has kv blocks for part of prefill request, so, provide them
|
||||
# as an external token count to scheduler.
|
||||
# The tokens will be loaded if not already present
|
||||
# in the prefill node local cache
|
||||
remote_num_tokens = params.get("remote_num_tokens") or 0
|
||||
count = (
|
||||
min(remote_num_tokens, request.num_prompt_tokens) - num_computed_tokens
|
||||
)
|
||||
if count > 0:
|
||||
# Check kv_recompute_threshold: skip pull if
|
||||
# remote tokens are below the threshold.
|
||||
if (
|
||||
self.kv_recompute_threshold > 0
|
||||
and count < self.kv_recompute_threshold
|
||||
):
|
||||
logger.debug(
|
||||
"Skipping remote pull for %s: %d remote tokens < threshold %d",
|
||||
request.request_id,
|
||||
count,
|
||||
self.kv_recompute_threshold,
|
||||
)
|
||||
return 0, False
|
||||
return count, True
|
||||
|
||||
# No remote prefill for this request.
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NIXLConnector update_state_after_alloc: "
|
||||
"num_external_tokens=%s, kv_transfer_params=%s",
|
||||
num_external_tokens,
|
||||
params,
|
||||
)
|
||||
|
||||
if not params:
|
||||
return
|
||||
|
||||
if params.get("do_remote_decode") or (
|
||||
params.get("do_remote_prefill") and self.is_bidirectional_kv_xfer_enabled
|
||||
):
|
||||
self._reqs_in_batch.add(request.request_id)
|
||||
if self.use_host_buffer and params.get("do_remote_decode"):
|
||||
# NOTE: when accelerator is not directly supported by Nixl,
|
||||
# prefilled blocks need to be saved to host memory before transfer.
|
||||
self._reqs_need_save[request.request_id] = request
|
||||
elif params.get("do_remote_prefill") or (
|
||||
params.get("do_remote_decode")
|
||||
and self.is_bidirectional_kv_xfer_enabled
|
||||
and not params.get("_remote_blocks_processed")
|
||||
):
|
||||
if params.get("remote_block_ids"):
|
||||
if all(
|
||||
p in params
|
||||
for p in (
|
||||
"remote_engine_id",
|
||||
"remote_request_id",
|
||||
"remote_host",
|
||||
"remote_port",
|
||||
)
|
||||
):
|
||||
# If remote_blocks and num_external_tokens = 0, we have
|
||||
# a full prefix cache hit on the local node. We need to call
|
||||
# send_notif in _read_blocks to free the memory on the remote node.
|
||||
|
||||
unhashed_local_block_ids: BlockIds = (
|
||||
blocks.get_unhashed_block_ids_all_groups()
|
||||
if num_external_tokens > 0
|
||||
else ()
|
||||
)
|
||||
local_block_ids = self.get_exchange_clipped_blocks(
|
||||
unhashed_local_block_ids
|
||||
)
|
||||
# Blocks covered by the local prefix cache, per KV cache group.
|
||||
# Each count fixes where that group's DCP slice starts, which the
|
||||
# worker needs to line up with the remote's slice.
|
||||
local_num_computed_blocks = tuple(
|
||||
sum(
|
||||
block.block_hash is not None and not block.is_null
|
||||
for block in group
|
||||
)
|
||||
for group in blocks.blocks
|
||||
)
|
||||
|
||||
# Get unhashed blocks to pull from remote. Mind that a full prefix
|
||||
# cache hit is indicated with an empty list.
|
||||
self._reqs_need_recv[request.request_id] = (
|
||||
request,
|
||||
local_block_ids,
|
||||
local_num_computed_blocks,
|
||||
# Parked in WAITING_FOR_REMOTE_KVS only when there is
|
||||
# something to pull; a full local hit stays RUNNING.
|
||||
num_external_tokens > 0,
|
||||
)
|
||||
|
||||
else:
|
||||
logger.warning(
|
||||
"Got invalid KVTransferParams: %s. This "
|
||||
"request will not utilize KVTransfer",
|
||||
params,
|
||||
)
|
||||
else:
|
||||
assert num_external_tokens == 0
|
||||
# Only trigger 1 KV transfer per request.
|
||||
params["do_remote_prefill"] = False
|
||||
params["_remote_blocks_processed"] = True
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: "BlockIds",
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Once a request is finished, determine whether request blocks
|
||||
should be freed now or will be sent asynchronously and freed later.
|
||||
"""
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NIXLConnector request_finished(%s), request_status=%s, "
|
||||
"kv_transfer_params=%s",
|
||||
request.request_id,
|
||||
request.status,
|
||||
params,
|
||||
)
|
||||
if not params:
|
||||
return False, None
|
||||
|
||||
is_p_node = bool(params.get("do_remote_decode"))
|
||||
is_d_node = not is_p_node
|
||||
|
||||
# Stop heartbeating for aborted requests that never reached finished_recving:
|
||||
# normal path cleans up in update_connector_output.
|
||||
self._stop_heartbeat(request.request_id)
|
||||
|
||||
if params.get("do_remote_prefill"):
|
||||
# If do_remote_prefill is still True when the request is finished,
|
||||
# update_state_after_alloc must not have been called (the request
|
||||
# must have been aborted before it was scheduled, e.g. via the
|
||||
# abort_immediately path used to clean up KV-transfer requests
|
||||
# rejected at the D-side serving layer).
|
||||
# To avoid stranding the prefill blocks in the prefill instance,
|
||||
# we must add empty block_ids to _reqs_need_recv so that our
|
||||
# worker side will notify and free blocks in the prefill instance.
|
||||
self._reqs_need_recv[request.request_id] = (request, [], (), False)
|
||||
params["do_remote_prefill"] = False
|
||||
return False, None
|
||||
|
||||
if is_d_node and not self.is_bidirectional_kv_xfer_enabled:
|
||||
return False, None
|
||||
|
||||
if request.status not in (
|
||||
RequestStatus.FINISHED_LENGTH_CAPPED,
|
||||
RequestStatus.FINISHED_STOPPED,
|
||||
):
|
||||
# Also include the case of a P/D Prefill request with immediate
|
||||
# block free (eg abort). Stop tracking this request.
|
||||
self._reqs_not_processed.add(request.request_id)
|
||||
# Clear _reqs_need_save if a request is aborted as partial prefill.
|
||||
self._reqs_need_save.pop(request.request_id, None)
|
||||
return False, None
|
||||
|
||||
# TODO: check whether block_ids actually ever be 0. If not we could
|
||||
# remove the conditional below
|
||||
delay_free_blocks = any(len(group) > 0 for group in block_ids)
|
||||
remote_num_tokens = 0
|
||||
blocks_expiry_time = None
|
||||
if delay_free_blocks:
|
||||
# Prefill request on remote. It will be read from D upon completion
|
||||
request_kv_blocks_ttl = self._kv_lease_duration
|
||||
if is_d_node:
|
||||
# For blocks pinned on D, use a simpler timeout for now instead of a
|
||||
# lease mechanism as turn2 request is client-driven.
|
||||
request_kv_blocks_ttl = self.decoder_kv_blocks_ttl
|
||||
logger.debug(
|
||||
"NIXLConnector request_finished(%s) waiting for %d seconds "
|
||||
"before releasing blocks",
|
||||
request.request_id,
|
||||
request_kv_blocks_ttl,
|
||||
)
|
||||
self._reqs_need_send[request.request_id] = (
|
||||
time.perf_counter() + request_kv_blocks_ttl
|
||||
)
|
||||
if is_d_node:
|
||||
# D blocks expiry time exported for the turn-2 readback.
|
||||
blocks_expiry_time = self._reqs_need_send[request.request_id]
|
||||
# NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones),
|
||||
# trimming down after allocating for the whole sequence length. Empty
|
||||
# blocks are always at the start of the list.
|
||||
# Here we "unpad" blocks to send the actual remote blocks to be read.
|
||||
block_ids = self.get_exchange_clipped_blocks(block_ids)
|
||||
|
||||
remote_num_tokens = request.num_computed_tokens
|
||||
|
||||
return delay_free_blocks, dict(
|
||||
do_remote_prefill=is_p_node,
|
||||
do_remote_decode=is_d_node,
|
||||
remote_block_ids=block_ids,
|
||||
remote_engine_id=self.engine_id,
|
||||
remote_request_id=request.request_id,
|
||||
remote_host=self.side_channel_host,
|
||||
remote_port=self.side_channel_port,
|
||||
tp_size=self.transfer_tp_size,
|
||||
dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size,
|
||||
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
|
||||
remote_num_tokens=remote_num_tokens,
|
||||
remote_blocks_expiry_time=blocks_expiry_time,
|
||||
transfer_mode=self._TRANSFER_MODE,
|
||||
)
|
||||
@@ -1,693 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pull-specific (READ) worker-side logic for the NIXL connector."""
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
|
||||
NixlBaseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlConnectorMetadata,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
ReadSpec,
|
||||
_is_attention_spec,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Slack (seconds) subtracted from D's exported block-expiry deadline on the turn-2
|
||||
# readback, absorbing clock-offset error and read latency.
|
||||
_KV_BLOCKS_EXPIRY_SAFETY_MARGIN = 5.0
|
||||
|
||||
|
||||
class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
"""Pull-specific (READ) worker logic."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
engine_id: str,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
|
||||
def start_load_kv(self, metadata: NixlConnectorMetadata):
|
||||
"""Start loading by triggering non-blocking nixl_xfer.
|
||||
We check for these trnxs to complete in each step().
|
||||
"""
|
||||
for req_id, meta in metadata.reqs_to_recv.items():
|
||||
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
|
||||
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
assert meta.remote is not None
|
||||
# Remote block IDs are kept logical here; expanded in
|
||||
# _read_blocks_for_req using the remote engine's phys ratio.
|
||||
remote_engine_id = meta.remote.engine_id
|
||||
logger.debug(
|
||||
"start_load_kv for request %s from remote engine %s. "
|
||||
"Num local_block_ids: %s. Num remote_block_ids: %s. ",
|
||||
req_id,
|
||||
remote_engine_id,
|
||||
len(meta.local_physical_block_ids),
|
||||
len(meta.remote.block_ids),
|
||||
)
|
||||
# Full local hits and aborted cleanup only notify P; no recv is awaited.
|
||||
# On a full local hit:
|
||||
# - Notification failure must not fail the request: its KV is local.
|
||||
# - Receive completion must not be reported: num_external_tokens == 0,
|
||||
# so the scheduler never entered WAITING_FOR_REMOTE_KVS to begin with.
|
||||
# Aborted cleanup requests have already been removed from the scheduler.
|
||||
if meta.awaiting_kvs or any(meta.local_block_ids):
|
||||
self._recving_metadata[req_id] = meta
|
||||
self._read_blocks_for_req(req_id, meta)
|
||||
|
||||
# Start transfers for requests whose handshakes have now finished.
|
||||
while not self._ready_requests.empty():
|
||||
self._read_blocks_for_req(*self._ready_requests.get_nowait())
|
||||
|
||||
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
|
||||
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
|
||||
# has nothing to send. Report the requests as sent right away so
|
||||
# the scheduler-side aggregation (world_size workers, and any
|
||||
# sibling connector inside a MultiConnector) still completes.
|
||||
self._replicated_pcp_done_sending.update(metadata.reqs_to_send)
|
||||
return
|
||||
|
||||
# Keep around the requests that have been part of a batch. This is
|
||||
# needed because async scheduling pushes the misalignment between the
|
||||
# moment in which requests expiration is set (P side) and the moment in
|
||||
# which blocks are read from D. As P can now more easily lag behind D
|
||||
# while processing the next batch, we make sure to only set an
|
||||
# expiration for requests that have not been read from D yet.
|
||||
for req_id in metadata.reqs_in_batch:
|
||||
self._reqs_to_process.add(req_id)
|
||||
|
||||
# Remove all requests that are not to be processed (eg aborted).
|
||||
for req_id in metadata.reqs_not_processed:
|
||||
self._reqs_to_process.discard(req_id)
|
||||
# We should never get an abort after setting an expiry timer
|
||||
assert req_id not in self._reqs_to_send
|
||||
|
||||
# Add to requests that are waiting to be read and track expiration.
|
||||
# Deadlines are stamped with the scheduler process's perf_counter,
|
||||
# which is not comparable to ours when the worker runs in another
|
||||
# process on another node (perf_counter epochs differ by boot time).
|
||||
# Rebase the remaining TTL onto our clock; broadcast latency only
|
||||
# lengthens the lease, which is the safe direction. A cross-node
|
||||
# epoch gap larger than the TTL otherwise expires the lease on
|
||||
# arrival and the blocks are freed before D reads them.
|
||||
now_local = time.perf_counter()
|
||||
for req_id, expiration_time in metadata.reqs_to_send.items():
|
||||
if req_id in self._reqs_to_process:
|
||||
if metadata.scheduler_clock:
|
||||
expiration_time = now_local + (
|
||||
expiration_time - metadata.scheduler_clock
|
||||
)
|
||||
self._reqs_to_send[req_id] = expiration_time
|
||||
|
||||
# Send heartbeats to P-side engines to keep KV blocks alive while
|
||||
# requests sit in the D scheduler WAITING queue.
|
||||
self._send_heartbeats(metadata)
|
||||
|
||||
def _is_turn2_read_expired(self, meta: ReqMeta) -> bool:
|
||||
"""Whether D's cached blocks for this turn-2 readback have (nearly) expired."""
|
||||
assert meta.remote is not None
|
||||
blocks_expiry_time = meta.remote.blocks_expiry_time
|
||||
# Deadline may be absent (router may not forward it) -> read as usual.
|
||||
if blocks_expiry_time is None or not meta.local_physical_block_ids:
|
||||
return False
|
||||
clock_offset = self._engine_clock_offset[meta.remote.engine_id]
|
||||
deadline = blocks_expiry_time - clock_offset
|
||||
return time.perf_counter() + _KV_BLOCKS_EXPIRY_SAFETY_MARGIN >= deadline
|
||||
|
||||
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
|
||||
assert meta.remote is not None and self.transfer_topo is not None
|
||||
engine_id = meta.remote.engine_id
|
||||
if engine_id in self._invalid_remote_engines:
|
||||
self._handle_failed_transfer(
|
||||
req_id, None, self._recv_failures, record_failed_transfer=False
|
||||
)
|
||||
return
|
||||
if engine_id not in self._remote_agents:
|
||||
# Also re-handshakes queued requests that outlived invalid-peer cleanup.
|
||||
self._background_nixl_handshake(req_id, engine_id, meta)
|
||||
return
|
||||
# Update last activity from this remote. Mind that cleanup is done on main
|
||||
# thread (this one), so we don't race on this structure.
|
||||
self._engine_last_active[engine_id] = time.perf_counter()
|
||||
|
||||
if self._bidirectional_kv_xfer_enabled and self._is_turn2_read_expired(meta):
|
||||
logger.warning(
|
||||
"Declining expired remote read for %s from engine %s.",
|
||||
req_id,
|
||||
engine_id,
|
||||
)
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
# KV expiry is reported separately from transport failures, so only
|
||||
# the state cleanup side of _handle_failed_transfer runs here.
|
||||
self._handle_failed_transfer(
|
||||
req_id, None, self._recv_failures, record_failed_transfer=False
|
||||
)
|
||||
return
|
||||
|
||||
if any(len(group) > 0 for group in meta.local_block_ids):
|
||||
# The scheduler waits for finished_recving from *every* worker.
|
||||
# Under DCP a rank's slice can legitimately come out empty when its
|
||||
# interleaved positions fall past the end of the sequence. _read_blocks
|
||||
# then takes the notify-only path without registering a transfer.
|
||||
# Seed the entry so this rank still reports completion.
|
||||
self._recving_transfers.setdefault(req_id, [])
|
||||
|
||||
plan = self.tp_mappings[engine_id]
|
||||
remote_info = self.transfer_topo.get_engine_info(engine_id)
|
||||
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
|
||||
|
||||
dcp_active = self.dcp_size > 1 or remote_info.remote_dcp_size > 1
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
remote_region_groups = self.dst_region_group_ids[engine_id]
|
||||
local_region_groups = self.region_group_ids or remote_region_groups
|
||||
if not local_block_ids:
|
||||
# Region expansion cannot index empty groups. Pass empty specs to
|
||||
# _read_blocks so its existing cache-hit notification path runs.
|
||||
read_specs = [
|
||||
ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[])
|
||||
for rank in plan.all_source_ranks
|
||||
]
|
||||
elif local_region_groups != remote_region_groups:
|
||||
if not self.use_mla or self._has_mamba:
|
||||
raise NotImplementedError(
|
||||
"Different NIXL cache-group layouts are only supported for "
|
||||
"pure MLA models"
|
||||
)
|
||||
if self.block_size != remote_info.remote_block_size:
|
||||
raise NotImplementedError(
|
||||
"Region-mapped NIXL transfers require matching physical block sizes"
|
||||
)
|
||||
remote_physical_block_ids = self._logical_to_kernel_block_ids(
|
||||
meta.remote.block_ids,
|
||||
remote_info.remote_physical_blocks_per_logical,
|
||||
)
|
||||
remote_by_region = self._block_ids_by_region(
|
||||
remote_physical_block_ids, remote_region_groups
|
||||
)
|
||||
local_by_region = self._block_ids_by_region(
|
||||
local_block_ids, local_region_groups
|
||||
)
|
||||
num_computed_blocks = None
|
||||
num_remote_blocks = None
|
||||
if (
|
||||
meta.remote.num_tokens is not None
|
||||
and meta.local_num_computed_blocks
|
||||
and all(group >= 0 for group in local_region_groups)
|
||||
and all(group >= 0 for group in remote_region_groups)
|
||||
):
|
||||
transfer_groups = self.kv_cache_config.transfer_group_ids
|
||||
num_computed_blocks = [
|
||||
meta.local_num_computed_blocks[transfer_groups[group]]
|
||||
* self._physical_blocks_per_logical_kv_block
|
||||
for group in local_region_groups
|
||||
]
|
||||
num_remote_blocks = cdiv(
|
||||
meta.remote.num_tokens, remote_info.remote_block_size
|
||||
)
|
||||
elif any(local_by_region) and (
|
||||
dcp_active
|
||||
or remote_info.remote_physical_blocks_per_logical
|
||||
!= self._physical_blocks_per_logical_kv_block
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Region-mapped pulls with DCP or different logical block sizes "
|
||||
"require remote_num_tokens, per-group prefix counts "
|
||||
"and unshared regions"
|
||||
)
|
||||
read_specs = [
|
||||
ReadSpec(
|
||||
rank,
|
||||
*self._apply_prefix_caching_by_region(
|
||||
local_by_region,
|
||||
remote_by_region,
|
||||
num_computed_blocks=num_computed_blocks,
|
||||
num_remote_blocks=num_remote_blocks,
|
||||
remote_rank=rank,
|
||||
remote_dcp_size=remote_info.remote_dcp_size,
|
||||
),
|
||||
block_ids_by_region=True,
|
||||
)
|
||||
for rank in plan.all_source_ranks
|
||||
]
|
||||
meta.region_blocks_to_zero = [
|
||||
list(blocks[sum(len(spec.local_block_ids[r]) for spec in read_specs) :])
|
||||
for r, blocks in enumerate(local_by_region)
|
||||
]
|
||||
else:
|
||||
remote_logical_block_ids = meta.remote.block_ids
|
||||
meta.remote.block_ids = self._logical_to_kernel_block_ids(
|
||||
remote_logical_block_ids,
|
||||
remote_info.remote_physical_blocks_per_logical,
|
||||
)
|
||||
num_groups = len(meta.local_block_ids)
|
||||
|
||||
def group_ids(block_ids: BlockIds, rank: int) -> list[list[int]]:
|
||||
return [
|
||||
list(block_ids[g]) if rank in plan.source_ranks_per_group[g] else []
|
||||
for g in range(num_groups)
|
||||
]
|
||||
|
||||
read_specs = []
|
||||
for rank in plan.all_source_ranks:
|
||||
if dcp_active:
|
||||
local_ids = group_ids(meta.local_block_ids, rank)
|
||||
remote_ids = group_ids(remote_logical_block_ids, rank)
|
||||
for g in range(num_groups):
|
||||
if not local_ids[g] or not _is_attention_spec(
|
||||
self._group_spec_types[g]
|
||||
):
|
||||
continue
|
||||
local_ids[g], remote_ids[g] = self._apply_dcp_prefix_caching(
|
||||
local_ids[g],
|
||||
remote_ids[g],
|
||||
remote_rank=rank,
|
||||
local_dcp_size=self.dcp_size,
|
||||
local_dcp_rank=self.dcp_rank,
|
||||
remote_dcp_size=remote_info.remote_dcp_size,
|
||||
local_num_computed_blocks=(
|
||||
meta.local_num_computed_blocks[g]
|
||||
),
|
||||
)
|
||||
local_physical_ids = self._logical_to_kernel_block_ids(
|
||||
local_ids, self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
remote_physical_ids = self._logical_to_kernel_block_ids(
|
||||
remote_ids,
|
||||
remote_info.remote_physical_blocks_per_logical,
|
||||
)
|
||||
else:
|
||||
local_physical_ids = group_ids(meta.local_physical_block_ids, rank)
|
||||
remote_physical_ids = group_ids(meta.remote.block_ids, rank)
|
||||
read_specs.append(
|
||||
ReadSpec(
|
||||
remote_rank=rank,
|
||||
local_block_ids=local_physical_ids,
|
||||
remote_block_ids=remote_physical_ids,
|
||||
)
|
||||
)
|
||||
|
||||
# D may have to perform multiple reads from different remote ranks.
|
||||
# Pure MLA reads once because its cache is replicated. Hybrid
|
||||
# MLA+SSM still needs one read per SSM source rank. With DCP, pure
|
||||
# MLA may also read from multiple ranks (disjoint token slices).
|
||||
if self.use_mla and tp_ratio < 0 and not self._has_mamba and not dcp_active:
|
||||
assert len(read_specs) == 1
|
||||
|
||||
for i, spec in enumerate(read_specs):
|
||||
remote_block_size = remote_info.remote_block_size
|
||||
logger.debug(
|
||||
"Remote agent %s available, calling _read_blocks"
|
||||
" on remote rank %s with remote block size %s for req %s",
|
||||
meta.remote.engine_id,
|
||||
spec.remote_rank,
|
||||
remote_block_size,
|
||||
req_id,
|
||||
)
|
||||
# Get side handles.
|
||||
if tp_ratio < 0 and (not self.use_mla or len(read_specs) > 1):
|
||||
# Remote tp_size > local tp_size: we must perform multiple
|
||||
# reads. Get the memory chunk onto which we will write to.
|
||||
split_key = (tp_ratio, remote_block_size)
|
||||
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
|
||||
local_dram_handle = (
|
||||
self._dram_src_handles_by_tp_ratio[split_key][i]
|
||||
if self._mixed_mem_types
|
||||
else None
|
||||
)
|
||||
else:
|
||||
# Single read from remote, we write to the whole memory region.
|
||||
# Also handle remote block size different from local block size.
|
||||
local_xfer_side_handle = self.src_xfer_handles_by_block_size[
|
||||
remote_block_size
|
||||
]
|
||||
local_dram_handle = (
|
||||
self._dram_src_handles_by_block_size[remote_block_size]
|
||||
if self._mixed_mem_types
|
||||
else None
|
||||
)
|
||||
|
||||
# Destination handle: remote_engine_id -> remote_rank -> handle.
|
||||
remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][
|
||||
spec.remote_rank
|
||||
]
|
||||
|
||||
# Once a read routes the request to failure reporting, the
|
||||
# scheduler may free and reuse its blocks, so no sibling READs
|
||||
# may be posted (and P must not be notified).
|
||||
if not self._read_blocks(
|
||||
read_spec=spec,
|
||||
request_id=req_id,
|
||||
dst_engine_id=meta.remote.engine_id,
|
||||
remote_request_id=meta.remote.request_id,
|
||||
local_xfer_side_handle=local_xfer_side_handle,
|
||||
local_dram_handle=local_dram_handle,
|
||||
remote_xfer_side_handle=remote_xfer_side_handle,
|
||||
expected_consumers=plan.local_consumers,
|
||||
awaiting_kvs=meta.awaiting_kvs,
|
||||
):
|
||||
return
|
||||
|
||||
if self.use_mla and tp_ratio < 0 and len(read_specs) == 1:
|
||||
# ..but we still need to notify the other remote ranks that we
|
||||
# have the blocks we need so they can update the request state.
|
||||
# Same thing for DCP (tp_size == dcp_size), so the raw tp_ratio already
|
||||
# reflects whether any remote replica is left unchosen.
|
||||
notif_id = f"{meta.remote.request_id}:{plan.local_consumers}".encode()
|
||||
remote_agents = self._remote_agents[meta.remote.engine_id]
|
||||
for rank_to_notify, agent in remote_agents.items():
|
||||
if rank_to_notify != (0, read_specs[0].remote_rank):
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="notification_failed",
|
||||
msg="Remote rank will not update request state. "
|
||||
"This may indicate network issues.",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
dst_engine_id=meta.remote.engine_id,
|
||||
remote_rank=rank_to_notify[1],
|
||||
remote_agent_name=agent,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
|
||||
def _read_blocks(
|
||||
self,
|
||||
read_spec: ReadSpec,
|
||||
dst_engine_id: str,
|
||||
request_id: str,
|
||||
remote_request_id: str,
|
||||
local_xfer_side_handle: int,
|
||||
local_dram_handle: int | None,
|
||||
remote_xfer_side_handle: int,
|
||||
expected_consumers: int,
|
||||
awaiting_kvs: bool,
|
||||
) -> bool:
|
||||
"""Post a READ point-to-point xfer request from a single local worker to
|
||||
a single remote worker.
|
||||
|
||||
Returns True when the read was posted (or was unnecessary), False
|
||||
when the request was routed to failure reporting — the caller must
|
||||
not post further transfers for it.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
remote_rank = read_spec.remote_rank
|
||||
local_block_ids = read_spec.local_block_ids
|
||||
remote_block_ids = read_spec.remote_block_ids
|
||||
|
||||
remote_info = self.transfer_topo.get_engine_info(dst_engine_id)
|
||||
block_size_ratio = self.transfer_topo.block_size_ratio(
|
||||
remote_info.remote_block_size
|
||||
)
|
||||
if block_size_ratio > 1:
|
||||
if read_spec.block_ids_by_region:
|
||||
raise NotImplementedError(
|
||||
"Region-mapped NIXL transfers require matching physical block sizes"
|
||||
)
|
||||
local_block_ids, remote_block_ids = (
|
||||
self._map_block_ids_for_block_size_ratio(
|
||||
local_block_ids, remote_block_ids, block_size_ratio
|
||||
)
|
||||
)
|
||||
# NOTE(rob): having the staging blocks be on the READER side is
|
||||
# not going to work well (since we will have to call rearrange tensors).
|
||||
# after we detect the txn is complete (which means we cannot make the
|
||||
# read trxn async easily). If we want to make "READ" happen cleanly,
|
||||
# then we will need to have the staging blocks on the remote side.
|
||||
|
||||
# NOTE(rob): according to nvidia the staging blocks are used to
|
||||
# saturate IB with heterogeneous TP sizes.
|
||||
|
||||
# Number of local workers that will notify this producer worker.
|
||||
# Propagate on notification so dst worker can wait before freeing.
|
||||
notif_id = f"{remote_request_id}:{expected_consumers}".encode()
|
||||
|
||||
# Full prefix cache hit: do not need to read remote blocks,
|
||||
# just notify P worker that we have the blocks we need.
|
||||
if not any(len(group) > 0 for group in local_block_ids):
|
||||
# A full prefix cache hit is indicated with an empty list.
|
||||
agent_name = self._remote_agents[dst_engine_id][(0, remote_rank)]
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="notification_failed",
|
||||
msg="P worker blocks will be freed after timeout. "
|
||||
"This may indicate network issues.",
|
||||
req_id=request_id,
|
||||
error=e,
|
||||
dst_engine_id=dst_engine_id,
|
||||
remote_rank=remote_rank,
|
||||
remote_agent_name=agent_name,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
# Report even on notification failure: the KV is already local, and
|
||||
# an unreported parked request would hold its blocks forever.
|
||||
# Notify-only recvs must stay unreported (scheduler asserts).
|
||||
if awaiting_kvs:
|
||||
self._recving_transfers.setdefault(request_id, [])
|
||||
return True
|
||||
|
||||
if not read_spec.block_ids_by_region:
|
||||
assert (
|
||||
len(remote_block_ids)
|
||||
== len(local_block_ids)
|
||||
== len(self.kv_cache_config.transfer_groups)
|
||||
)
|
||||
if not (self.dcp_size > 1 or remote_info.remote_dcp_size > 1):
|
||||
local_block_ids, remote_block_ids = self._apply_prefix_caching(
|
||||
decode_block_ids=local_block_ids,
|
||||
prefill_block_ids=remote_block_ids,
|
||||
decode_physical_per_logical=(
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
),
|
||||
prefill_physical_per_logical=(
|
||||
remote_info.remote_physical_blocks_per_logical
|
||||
),
|
||||
)
|
||||
|
||||
# NOTE (nicolo) With homogeneous TP, each TP worker loads KV from
|
||||
# corresponding rank. With heterogeneous TP, fixing D>P, the D tp
|
||||
# workers will issue xfers to parts of the P worker remote kv caches.
|
||||
|
||||
# Get descs ids.
|
||||
remote_block_descs_ids = self._compute_desc_ids(
|
||||
block_ids=remote_block_ids,
|
||||
dst_num_blocks=self.dst_num_blocks[dst_engine_id],
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical,
|
||||
region_num_blocks=(self.dst_region_num_blocks.get(dst_engine_id) or None),
|
||||
region_group_ids=(
|
||||
list(range(self.num_regions))
|
||||
if read_spec.block_ids_by_region
|
||||
else (self.dst_region_group_ids.get(dst_engine_id) or None)
|
||||
),
|
||||
uses_region_group_mapping=(
|
||||
self.num_regions > 1
|
||||
if read_spec.block_ids_by_region
|
||||
else self.dst_uses_region_group_mapping[dst_engine_id]
|
||||
),
|
||||
)
|
||||
local_block_descs_ids = self._compute_desc_ids(
|
||||
block_ids=local_block_ids,
|
||||
dst_num_blocks=self.dst_num_blocks[self.engine_id],
|
||||
block_size_ratio=block_size_ratio,
|
||||
physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block,
|
||||
region_num_blocks=(self.dst_region_num_blocks.get(self.engine_id) or None),
|
||||
region_group_ids=(
|
||||
list(range(self.num_regions))
|
||||
if read_spec.block_ids_by_region
|
||||
else (self.region_group_ids or None)
|
||||
),
|
||||
uses_region_group_mapping=(
|
||||
self.num_regions > 1
|
||||
if read_spec.block_ids_by_region
|
||||
else self._uses_region_group_mapping
|
||||
),
|
||||
)
|
||||
|
||||
assert len(local_block_descs_ids) == len(remote_block_descs_ids)
|
||||
|
||||
# Prepare transfer with Nixl.
|
||||
handle = None
|
||||
try:
|
||||
if self._mixed_mem_types:
|
||||
self._read_blocks_mixed(
|
||||
request_id=request_id,
|
||||
local_block_size_key=remote_info.remote_block_size,
|
||||
local_device_handle=local_xfer_side_handle,
|
||||
local_dram_handle=local_dram_handle,
|
||||
remote_xfer_side_handle=remote_xfer_side_handle,
|
||||
local_block_descs_ids=local_block_descs_ids,
|
||||
remote_block_descs_ids=remote_block_descs_ids,
|
||||
notif_agent=self._remote_agents[dst_engine_id][(0, remote_rank)],
|
||||
notif_id=notif_id,
|
||||
)
|
||||
return True
|
||||
handle = self.nixl_wrapper.make_prepped_xfer(
|
||||
"READ",
|
||||
local_xfer_side_handle,
|
||||
local_block_descs_ids,
|
||||
remote_xfer_side_handle,
|
||||
remote_block_descs_ids,
|
||||
notif_msg=notif_id,
|
||||
)
|
||||
|
||||
# Begin async xfer.
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
|
||||
# Use handle to check completion in future step().
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
return True
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="transfer_setup_failed",
|
||||
req_id=request_id,
|
||||
msg="Deferring failure reporting until outstanding transfers finish",
|
||||
error=e,
|
||||
dst_engine_id=dst_engine_id,
|
||||
remote_rank=remote_rank,
|
||||
)
|
||||
if not self._handle_failed_transfer(
|
||||
request_id, handle, self._recv_failures
|
||||
):
|
||||
assert handle is not None
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
return False
|
||||
|
||||
def _read_blocks_mixed(
|
||||
self,
|
||||
request_id: str,
|
||||
local_block_size_key: int,
|
||||
local_device_handle: int,
|
||||
local_dram_handle: int | None,
|
||||
remote_xfer_side_handle: int,
|
||||
local_block_descs_ids: np.ndarray,
|
||||
remote_block_descs_ids: np.ndarray,
|
||||
notif_agent: str,
|
||||
notif_id: bytes,
|
||||
) -> None:
|
||||
"""Split a READ across the local DRAM and device descriptor lists."""
|
||||
desc_is_dram = self._desc_is_dram_by_block_size[local_block_size_key]
|
||||
desc_pos = self._desc_pos_by_block_size[local_block_size_key]
|
||||
local_ids = np.asarray(local_block_descs_ids)
|
||||
remote_ids = np.asarray(remote_block_descs_ids)
|
||||
is_dram = desc_is_dram[local_ids]
|
||||
|
||||
if self._skip_dram_xfer:
|
||||
# TP rank 0 reads the DRAM (shared host pool) part for every
|
||||
# local rank; keep only the device part. Register the request
|
||||
# even if nothing is left, so it still completes and notifies.
|
||||
keep = ~is_dram
|
||||
local_ids = local_ids[keep]
|
||||
remote_ids = remote_ids[keep]
|
||||
is_dram = is_dram[keep]
|
||||
self._recving_transfers.setdefault(request_id, [])
|
||||
assert local_dram_handle is not None or not is_dram.any()
|
||||
reads = (
|
||||
(is_dram, local_dram_handle),
|
||||
(~is_dram, local_device_handle),
|
||||
)
|
||||
handles: list[int] = []
|
||||
try:
|
||||
for mask, local_handle in reads:
|
||||
if mask.any():
|
||||
handles.append(
|
||||
self.nixl_wrapper.make_prepped_xfer(
|
||||
"READ",
|
||||
local_handle,
|
||||
desc_pos[local_ids[mask]],
|
||||
remote_xfer_side_handle,
|
||||
remote_ids[mask],
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
for handle in handles:
|
||||
if not self._try_release_xfer_handle(request_id, handle):
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
raise
|
||||
|
||||
self._pending_recv_notifs.setdefault(request_id, []).append(
|
||||
(notif_agent, notif_id)
|
||||
)
|
||||
for i, handle in enumerate(handles):
|
||||
try:
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
except Exception:
|
||||
for unstarted in handles[i:]:
|
||||
if not self._try_release_xfer_handle(request_id, unstarted):
|
||||
self._recving_transfers[request_id].append(unstarted)
|
||||
raise
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
|
||||
def _get_new_notifs(self) -> set[str]:
|
||||
"""Get req_ids which got a remote xfer message. When multiple consumers
|
||||
are reading from the same producer (heterogeneous TP or DCP
|
||||
scenario), wait for all consumers to be done pulling.
|
||||
|
||||
Also handles heartbeat notifications ("HB:req1,req2,...") by
|
||||
extending the lease on the referenced requests.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
notified_req_ids: set[str] = set()
|
||||
for notifs in self.nixl_wrapper.get_new_notifs().values():
|
||||
for notif in notifs:
|
||||
msg = notif.decode("utf-8")
|
||||
|
||||
# Handle heartbeat messages from D-side.
|
||||
if msg.startswith("HB:"):
|
||||
self._handle_heartbeat(msg[3:])
|
||||
continue
|
||||
|
||||
req_id, expected_consumers = msg.rsplit(":", 1)
|
||||
if (
|
||||
req_id not in self._reqs_to_send
|
||||
and req_id not in self._reqs_to_process
|
||||
):
|
||||
self.xfer_stats.record_notification_after_expiry()
|
||||
logger.error(
|
||||
"Potentially invalid KV blocks for "
|
||||
"unrecognized request %s were retrieved by "
|
||||
"a decode worker. They may have expired.",
|
||||
req_id,
|
||||
)
|
||||
continue
|
||||
|
||||
# Every reader of this req_id reports the same count (it's
|
||||
# derived from aggregate topology, not the specific rank),
|
||||
# so repeated notifications never disagree on it.
|
||||
self.expected_consumer_notifications_by_req[req_id] = int(
|
||||
expected_consumers
|
||||
)
|
||||
|
||||
self.consumer_notification_counts_by_req[req_id] += 1
|
||||
# Wait all consumers (D) to be done reading before freeing.
|
||||
if (
|
||||
self.consumer_notification_counts_by_req[req_id]
|
||||
== self.expected_consumer_notifications_by_req[req_id]
|
||||
):
|
||||
notified_req_ids.add(req_id)
|
||||
del self.consumer_notification_counts_by_req[req_id]
|
||||
del self.expected_consumer_notifications_by_req[req_id]
|
||||
self._reqs_to_process.remove(req_id)
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
return notified_req_ids
|
||||
@@ -1,357 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Push-specific scheduler-side logic for the NIXL connector.
|
||||
|
||||
In push mode, scheduler-side responsibilities are:
|
||||
|
||||
* D side (decode): on ``update_state_after_alloc``, stash registration data
|
||||
(D's identity + locally allocated block IDs) into
|
||||
``_push_pending_registrations``. The D worker drains it from
|
||||
``meta.push_registrations`` next step and sends a NIXL notification to the
|
||||
P worker (no scheduler-level networking).
|
||||
* P side (prefill): on ``request_finished``, stash the finished block IDs
|
||||
into ``_finished_request_blocks`` for the lease, and into
|
||||
``_newly_finished_push_blocks`` so the P worker picks them up via
|
||||
``meta.push_finished_blocks`` and matches against any D registrations
|
||||
it already received via NIXL notifications.
|
||||
* Both sides: ``has_pending_push_work`` keeps the engine main loop stepping
|
||||
while pushes are in flight. ``update_connector_output`` cleans up
|
||||
``_finished_request_blocks`` once the WRITE completes.
|
||||
|
||||
A soft per-registration watchdog on the D scheduler fails requests that have
|
||||
been registered but not fulfilled within a configurable timeout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
|
||||
NixlBaseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlConnectorMetadata,
|
||||
ReqId,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
"""Push-specific scheduler logic (WRITE-based KV transfer).
|
||||
|
||||
All P2P communication is deferred to the worker level via NIXL
|
||||
notifications. The scheduler communicates with workers only through
|
||||
the standard ``build_connector_meta`` / ``update_connector_output``
|
||||
hooks.
|
||||
"""
|
||||
|
||||
_TRANSFER_MODE: str = "push"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
engine_id: str,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
if self.is_bidirectional_kv_xfer_enabled:
|
||||
raise NotImplementedError(
|
||||
"Bidirectional KV transfer is not supported for NIXL push connector."
|
||||
)
|
||||
|
||||
# D-side: registration data to pass to D workers via metadata on
|
||||
# the next ``build_connector_meta`` call.
|
||||
self._push_pending_registrations: dict[ReqId, dict[str, Any]] = {}
|
||||
|
||||
# D-side: track the wall-clock deadline for each registered request
|
||||
# to detect "registered but never fulfilled" failures (e.g. the P
|
||||
# node disappeared after registration). Keyed by D request_id.
|
||||
self._push_registration_deadlines: dict[ReqId, float] = {}
|
||||
|
||||
# P-side: block IDs for finished requests, kept for the lease and
|
||||
# used to drive ``has_pending_push_work``.
|
||||
self._finished_request_blocks: dict[ReqId, BlockIds] = {}
|
||||
# P-side: newly finished blocks to ship to P workers on next step.
|
||||
self._newly_finished_push_blocks: dict[ReqId, BlockIds] = {}
|
||||
|
||||
# Soft watchdog timeout (seconds) for D-side registrations that
|
||||
# never receive a push completion. Defaults to the existing
|
||||
# decoder KV blocks TTL so behaviour matches the lease.
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self._push_registration_timeout: float = float(
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"push_registration_timeout",
|
||||
self.decoder_kv_blocks_ttl,
|
||||
)
|
||||
)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
"""In push mode, D doesn't pull — it registers blocks and waits.
|
||||
|
||||
However, we still need to handle the do_remote_prefill case where D
|
||||
needs to know how many tokens will be pushed.
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NixlPushConnector get_num_new_matched_tokens: "
|
||||
"num_computed_tokens=%s, kv_transfer_params=%s",
|
||||
num_computed_tokens,
|
||||
params,
|
||||
)
|
||||
|
||||
if params is not None and params.get("do_remote_prefill"):
|
||||
token_ids = request.prompt_token_ids or []
|
||||
actual = self._get_remote_prefill_token_count(len(token_ids))
|
||||
count = actual - num_computed_tokens
|
||||
if count > 0:
|
||||
return count, True
|
||||
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
|
||||
):
|
||||
"""In push mode, D stores registration data for the worker to send
|
||||
to P via NIXL notification (deferred to ``build_connector_meta``).
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NixlPushConnector update_state_after_alloc: "
|
||||
"num_external_tokens=%s, kv_transfer_params=%s",
|
||||
num_external_tokens,
|
||||
params,
|
||||
)
|
||||
|
||||
if not params:
|
||||
return
|
||||
|
||||
# P side: track the request as in-batch so the lease accounting
|
||||
# matches what the worker expects on the next step.
|
||||
if params.get("do_remote_decode"):
|
||||
self._reqs_in_batch.add(request.request_id)
|
||||
|
||||
# P side with host-buffer offload: defer save to the worker.
|
||||
if self.use_host_buffer and params.get("do_remote_decode"):
|
||||
self._reqs_need_save[request.request_id] = request
|
||||
return
|
||||
|
||||
# D side: only act on the first call (``do_remote_prefill`` is
|
||||
# unset on re-entry by the marker below).
|
||||
if not params.get("do_remote_prefill"):
|
||||
return
|
||||
|
||||
if num_external_tokens <= 0:
|
||||
# Nothing to receive: full prefix-cache hit on D, no
|
||||
# registration to stage.
|
||||
return
|
||||
|
||||
# First-pass D path: stash registration data the worker will
|
||||
# ship to P on the next ``build_connector_meta`` cycle.
|
||||
logger.debug(
|
||||
"KV PUSH mode: D node storing registration for request %s",
|
||||
request.request_id,
|
||||
)
|
||||
local_block_ids: BlockIds = blocks.get_unhashed_block_ids_all_groups()
|
||||
local_block_ids = self.get_exchange_clipped_blocks(local_block_ids)
|
||||
|
||||
# ``remote_*`` fields are P's coordinates (from D's perspective).
|
||||
# ``decode_*`` fields are D's own info that P needs for the
|
||||
# reverse handshake before WRITE-ing.
|
||||
self._push_pending_registrations[request.request_id] = {
|
||||
"request_id": request.request_id,
|
||||
"decode_engine_id": self.engine_id,
|
||||
"decode_host": self.side_channel_host,
|
||||
"decode_port": self.side_channel_port,
|
||||
"decode_tp_size": (self.vllm_config.parallel_config.tensor_parallel_size),
|
||||
"local_block_ids": local_block_ids,
|
||||
"remote_engine_id": params["remote_engine_id"],
|
||||
"remote_host": params["remote_host"],
|
||||
"remote_port": params["remote_port"],
|
||||
"remote_tp_size": params["tp_size"],
|
||||
"remote_pp_size": params.get("pp_size", 1),
|
||||
}
|
||||
self._push_registration_deadlines[request.request_id] = (
|
||||
time.perf_counter() + self._push_registration_timeout
|
||||
)
|
||||
# In push mode D doesn't know P's blocks; P determines them
|
||||
# from the registration. We still track the request as
|
||||
# needing recv so the engine waits for P's WRITE completion.
|
||||
# ``remote_block_ids`` is also seeded to an empty tuple so the
|
||||
# base scheduler's ``add_new_req_to_recv`` can build the
|
||||
# ReqMeta without a KeyError — the actual remote block IDs are
|
||||
# learned by P over the NIXL handshake at WRITE time.
|
||||
params["remote_block_ids"] = ()
|
||||
self._reqs_need_recv[request.request_id] = (request, local_block_ids, (), False)
|
||||
|
||||
# Mark as processed so a re-entry (e.g. preemption + reschedule)
|
||||
# doesn't re-stage the registration.
|
||||
params["do_remote_prefill"] = False
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: BlockIds,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Push-mode request_finished: stores blocks for workers."""
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NixlPushConnector request_finished(%s), request_status=%s, "
|
||||
"kv_transfer_params=%s",
|
||||
request.request_id,
|
||||
request.status,
|
||||
params,
|
||||
)
|
||||
if not params:
|
||||
return False, None
|
||||
|
||||
is_p_node = bool(params.get("do_remote_decode"))
|
||||
|
||||
self._stop_heartbeat(request.request_id)
|
||||
# Drop any pending registration deadline; the request either
|
||||
# completed or was cancelled.
|
||||
self._push_registration_deadlines.pop(request.request_id, None)
|
||||
|
||||
if params.get("do_remote_prefill"):
|
||||
# ``do_remote_prefill`` is still set, which means
|
||||
# ``update_state_after_alloc`` never ran (it would have
|
||||
# flipped this flag to False). The request was aborted
|
||||
# before it could be scheduled — e.g. rejected at the D
|
||||
# serving layer via abort_immediately. To keep P from
|
||||
# stranding the prefill blocks, we still register an empty
|
||||
# recv so the worker emits a notif that lets P free them.
|
||||
# Seed remote_block_ids so add_new_req_to_recv won't KeyError.
|
||||
params["remote_block_ids"] = ()
|
||||
self._reqs_need_recv[request.request_id] = (request, [], (), False)
|
||||
params["do_remote_prefill"] = False
|
||||
return False, None
|
||||
|
||||
# Push connector only acts on the P-side terminal path; D-side
|
||||
# finishing without a remote prefill is a no-op.
|
||||
if not is_p_node:
|
||||
return False, None
|
||||
|
||||
if request.status not in (
|
||||
RequestStatus.FINISHED_LENGTH_CAPPED,
|
||||
RequestStatus.FINISHED_STOPPED,
|
||||
):
|
||||
self._reqs_not_processed.add(request.request_id)
|
||||
self._reqs_need_save.pop(request.request_id, None)
|
||||
return False, None
|
||||
|
||||
delay_free_blocks = any(len(group) > 0 for group in block_ids)
|
||||
remote_num_tokens = 0
|
||||
if delay_free_blocks:
|
||||
logger.debug(
|
||||
"NixlPushConnector request_finished(%s) waiting for %d seconds "
|
||||
"before releasing blocks",
|
||||
request.request_id,
|
||||
self._kv_lease_duration,
|
||||
)
|
||||
self._reqs_need_send[request.request_id] = (
|
||||
time.perf_counter() + self._kv_lease_duration
|
||||
)
|
||||
|
||||
block_ids = self.get_exchange_clipped_blocks(block_ids)
|
||||
remote_num_tokens = request.num_computed_tokens
|
||||
|
||||
# Store finished blocks for worker-level matching with D
|
||||
# registrations (via NIXL notifications).
|
||||
self._finished_request_blocks[request.request_id] = block_ids
|
||||
self._newly_finished_push_blocks[request.request_id] = block_ids
|
||||
|
||||
return delay_free_blocks, dict(
|
||||
do_remote_prefill=True,
|
||||
do_remote_decode=False,
|
||||
remote_block_ids=block_ids,
|
||||
remote_engine_id=self.engine_id,
|
||||
remote_request_id=request.request_id,
|
||||
remote_host=self.side_channel_host,
|
||||
remote_port=self.side_channel_port,
|
||||
tp_size=self.transfer_tp_size,
|
||||
dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size,
|
||||
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
|
||||
remote_num_tokens=remote_num_tokens,
|
||||
transfer_mode=self._TRANSFER_MODE,
|
||||
)
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
meta = super().build_connector_meta(scheduler_output)
|
||||
assert isinstance(meta, NixlConnectorMetadata)
|
||||
|
||||
# Watchdog: any D-side registration whose deadline has passed without
|
||||
# a corresponding push completion is treated as failed and cleaned up.
|
||||
# The corresponding request is already tracked via _reqs_need_recv;
|
||||
# the engine layer will eventually time it out via the lease, but we
|
||||
# at least drop the stale registration so we don't keep retrying.
|
||||
now = time.perf_counter()
|
||||
# Deadlines are inserted in non-decreasing order (monotonic clock +
|
||||
# constant timeout, armed once per request), and dict insertion order
|
||||
# is preserved across key deletions, so we can stop at the first
|
||||
# not-yet-expired entry instead of scanning the whole dict.
|
||||
expired = []
|
||||
for rid, deadline in self._push_registration_deadlines.items():
|
||||
if deadline > now:
|
||||
break
|
||||
expired.append(rid)
|
||||
for rid in expired:
|
||||
self._push_registration_deadlines.pop(rid, None)
|
||||
# Avoid resending a registration that already timed out.
|
||||
self._push_pending_registrations.pop(rid, None)
|
||||
logger.warning(
|
||||
"NixlPushConnector: registration for request %s timed out "
|
||||
"after %.1fs without a push completion",
|
||||
rid,
|
||||
self._push_registration_timeout,
|
||||
)
|
||||
|
||||
# D side: package pending registrations for D workers to send out.
|
||||
if self._push_pending_registrations:
|
||||
meta.push_registrations = dict(self._push_pending_registrations)
|
||||
self._push_pending_registrations.clear()
|
||||
|
||||
# P side: package newly finished blocks for P workers to match against
|
||||
# any D registrations they have received via NIXL notifications.
|
||||
if self._newly_finished_push_blocks:
|
||||
meta.push_finished_blocks = dict(self._newly_finished_push_blocks)
|
||||
self._newly_finished_push_blocks.clear()
|
||||
|
||||
return meta
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
# Keep the engine main loop alive while we have:
|
||||
# - finished P blocks awaiting WRITE completion, or
|
||||
# - pending D registrations the worker has not yet shipped, or
|
||||
# - newly finished blocks not yet shipped to P workers.
|
||||
return bool(self._finished_request_blocks or self._push_pending_registrations)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Clean up finished request blocks after push completes."""
|
||||
super().update_connector_output(connector_output)
|
||||
for req_id in connector_output.finished_sending or ():
|
||||
self._finished_request_blocks.pop(req_id, None)
|
||||
# On D side, finished_recving means the push completed; clear the
|
||||
# watchdog so we don't trip an expiration on a fulfilled request.
|
||||
for req_id in connector_output.finished_recving or ():
|
||||
self._push_registration_deadlines.pop(req_id, None)
|
||||
@@ -1,827 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Push-specific (WRITE) worker-side logic for the NIXL connector.
|
||||
|
||||
A dedicated ``nixl-push-writer`` thread owns all push-related NIXL ops:
|
||||
calls ``get_new_notifs`` (routing PUSH_REG internally; HB / completion
|
||||
notifs are forwarded to the engine main thread), sends PUSH_REG via
|
||||
``send_notif``, matches D registrations with P finished blocks, and
|
||||
issues WRITE transfers via ``make_prepped_xfer`` / ``transfer``.
|
||||
|
||||
The engine main thread feeds the writer through queues:
|
||||
``_reg_send_inbox`` (D-side regs to send), ``_finished_blocks_inbox``
|
||||
(P-side blocks from metadata) and ``_pending_completion_notifs``
|
||||
(non-PUSH_REG notifs forwarded back for HB / completion accounting).
|
||||
The handshake-completion callback feeds ``_deferred_push_inbox`` with
|
||||
matched pushes whose P→D handshake has finished so the writer can
|
||||
(re-)issue the WRITE without ever blocking on the network.
|
||||
|
||||
Wake model: the writer self-polls every
|
||||
``_PUSH_WRITER_POLL_INTERVAL_MS`` only while it has unmatched
|
||||
``_push_finished_blocks`` (i.e. P-side blocks waiting for a D PUSH_REG
|
||||
notif that has no other wake source). All other progress is
|
||||
event-driven: the engine main thread sets ``_push_writer_wake`` from
|
||||
``start_load_kv`` (when handing it new work) and from ``get_finished``
|
||||
(so each engine step gives the writer a chance to drain NIXL notifs);
|
||||
the handshake-completion callback sets the same event after a deferred
|
||||
PUSH_REG send or a deferred push WRITE has been queued. When a request's
|
||||
lease expires (the base worker reports it via ``done_sending``) or the WRITE completes,
|
||||
``get_finished`` enqueues an eviction onto ``_evict_finished_inbox`` so
|
||||
the writer drops any leftover ``_push_finished_blocks`` /
|
||||
``_pending_d_registrations`` and stops self-polling.
|
||||
"""
|
||||
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import Future
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import msgspec
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
|
||||
NixlBaseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
PUSH_REG_NOTIF_PREFIX,
|
||||
NixlConnectorMetadata,
|
||||
RemoteMeta,
|
||||
ReqId,
|
||||
ReqMeta,
|
||||
TransferHandle,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
ReadSpec,
|
||||
_is_attention_spec,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import get_base_request_id
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Writer-thread poll cadence while there is in-flight push state. When
|
||||
# fully idle, the writer blocks on a wake event signalled by the engine
|
||||
# main thread (start_load_kv / get_finished). Smaller -> lower latency
|
||||
# while active, slightly more CPU.
|
||||
_PUSH_WRITER_POLL_INTERVAL_MS = 1.0
|
||||
|
||||
|
||||
class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
"""Push-specific (WRITE) worker logic. See module docstring."""
|
||||
|
||||
# Distinguishes push from pull in the NIXL compatibility hash.
|
||||
_TRANSFER_MODE: str = "push"
|
||||
|
||||
_supports_pp_hma = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
engine_id: str,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
|
||||
# Heartbeat handshakes to a PP-sharded producer must be notif-only,
|
||||
# like the PUSH_REG path.
|
||||
self._hb_handshake_notif_only = True
|
||||
|
||||
# Push-specific state.
|
||||
# P-side: outgoing WRITE handles awaiting completion, keyed by
|
||||
# request_id. Mutated by writer (submit) and main thread
|
||||
# (``_pop_done_transfers``); guarded by
|
||||
# ``_sending_transfers_lock``.
|
||||
self._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list)
|
||||
self._send_failures: set[ReqId] = set()
|
||||
self._sending_transfers_lock = threading.Lock()
|
||||
|
||||
# Writer-thread owned matching state.
|
||||
# P-side: finished request blocks received from scheduler metadata
|
||||
# that have not yet been matched with an incoming D registration.
|
||||
self._push_finished_blocks: dict[ReqId, BlockIds] = {}
|
||||
# P-side: D registrations received via NIXL notification that have
|
||||
# not yet been matched with a finished P request.
|
||||
self._pending_d_registrations: dict[ReqId, dict[str, Any]] = {}
|
||||
|
||||
# Cross-thread channels.
|
||||
self._reg_send_inbox: queue.Queue[tuple[str, dict[str, Any]]] = queue.Queue()
|
||||
self._finished_blocks_inbox: queue.Queue[tuple[str, BlockIds]] = queue.Queue()
|
||||
self._pending_completion_notifs: queue.Queue[bytes] = queue.Queue()
|
||||
# Main thread → writer: req_ids whose lease has expired or whose
|
||||
# WRITE has completed. Writer drops them from
|
||||
# ``_push_finished_blocks`` so an unmatched entry doesn't keep the
|
||||
# writer busy-polling forever.
|
||||
self._evict_finished_inbox: queue.Queue[str] = queue.Queue()
|
||||
# Handshakes that have just completed and are ready for the WRITE on wthread
|
||||
self._deferred_push_inbox = queue.Queue[tuple[str, BlockIds, dict[str, Any]]]()
|
||||
|
||||
# Wake signal from engine main thread (start_load_kv / get_finished).
|
||||
# Writer self-polls at _PUSH_WRITER_POLL_INTERVAL_MS while it has
|
||||
# active in-flight state; otherwise it blocks until signalled.
|
||||
self._push_writer_wake = threading.Event()
|
||||
|
||||
self._push_writer_stop = threading.Event()
|
||||
self._push_writer_thread: threading.Thread | None = None
|
||||
|
||||
# --- Lifecycle ----------------------------------------------------- #
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, "torch.Tensor"]):
|
||||
super().register_kv_caches(kv_caches)
|
||||
if self._mixed_mem_types:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector does not support mixed-memory KV caches"
|
||||
)
|
||||
if self._push_writer_thread is None:
|
||||
self._push_writer_thread = threading.Thread(
|
||||
target=self._push_writer_loop,
|
||||
daemon=True,
|
||||
name="nixl-push-writer",
|
||||
)
|
||||
self._push_writer_thread.start()
|
||||
logger.info("nixl-push-writer thread started (rank=%d)", self.tp_rank)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._push_writer_stop.set()
|
||||
# Unblock the writer if it's waiting in the no-active-state branch.
|
||||
self._push_writer_wake.set()
|
||||
if self._push_writer_thread is not None:
|
||||
self._push_writer_thread.join(timeout=2)
|
||||
self._push_writer_thread = None
|
||||
with self._sending_transfers_lock:
|
||||
for handles in self._sending_transfers.values():
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self._sending_transfers.clear()
|
||||
self._send_failures.clear()
|
||||
super().shutdown()
|
||||
|
||||
# --- Engine-main-thread entry point -------------------------------- #
|
||||
|
||||
def start_load_kv(self, metadata: NixlConnectorMetadata):
|
||||
"""Pre-process metadata; defer NIXL ops to the writer thread."""
|
||||
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
|
||||
return
|
||||
|
||||
# D-side: track reqs waiting for P to push.
|
||||
for req_id, meta in metadata.reqs_to_recv.items():
|
||||
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
|
||||
meta.local_block_ids, self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
assert meta.remote is not None
|
||||
remote_engine_id = meta.remote.engine_id
|
||||
# Update last activity from this remote (same as pull), but only for
|
||||
# an already-connected engine so we never leave an _engine_last_active
|
||||
# entry without a _remote_agents entry (breaks _cleanup_remote_engine).
|
||||
if remote_engine_id in self._remote_agents:
|
||||
self._engine_last_active[remote_engine_id] = time.perf_counter()
|
||||
|
||||
logger.debug(
|
||||
"start_load_kv (push) for request %s from remote engine %s. "
|
||||
"Num local_block_ids: %s. Num remote_block_ids: %s. ",
|
||||
req_id,
|
||||
remote_engine_id,
|
||||
len(meta.local_physical_block_ids),
|
||||
len(meta.remote.block_ids),
|
||||
)
|
||||
self._recving_metadata[req_id] = meta
|
||||
|
||||
# --- D-side: registrations to send to P via NIXL ---
|
||||
if metadata.push_registrations:
|
||||
for req_id, reg_data in metadata.push_registrations.items():
|
||||
self._reg_send_inbox.put((req_id, reg_data))
|
||||
self._push_writer_wake.set()
|
||||
|
||||
# --- P-side: newly finished blocks awaiting a D registration match ---
|
||||
if metadata.push_finished_blocks:
|
||||
for req_id, block_ids in metadata.push_finished_blocks.items():
|
||||
self._finished_blocks_inbox.put((req_id, block_ids))
|
||||
self._push_writer_wake.set()
|
||||
|
||||
# Batch + lease tracking (same as pull).
|
||||
for req_id in metadata.reqs_in_batch:
|
||||
self._reqs_to_process.add(req_id)
|
||||
for req_id in metadata.reqs_not_processed:
|
||||
self._reqs_to_process.discard(req_id)
|
||||
assert req_id not in self._reqs_to_send
|
||||
# Rebase scheduler-clock deadlines onto this worker's clock — see the
|
||||
# equivalent block in pull_worker.start_load_kv for the rationale.
|
||||
now_local = time.perf_counter()
|
||||
for req_id, expiration_time in metadata.reqs_to_send.items():
|
||||
if req_id in self._reqs_to_process:
|
||||
if metadata.scheduler_clock:
|
||||
expiration_time = now_local + (
|
||||
expiration_time - metadata.scheduler_clock
|
||||
)
|
||||
self._reqs_to_send[req_id] = expiration_time
|
||||
|
||||
# Heartbeats still leave from the main thread (base worker behaviour).
|
||||
self._send_heartbeats(metadata)
|
||||
|
||||
# --- Writer thread ------------------------------------------------- #
|
||||
|
||||
def _push_writer_loop(self) -> None:
|
||||
sleep_s = _PUSH_WRITER_POLL_INTERVAL_MS / 1000.0
|
||||
|
||||
while not self._push_writer_stop.is_set():
|
||||
try:
|
||||
# 1. D registrations to send.
|
||||
while True:
|
||||
try:
|
||||
rid, rd = self._reg_send_inbox.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._send_registration_to_p(rid, rd)
|
||||
|
||||
# 2. Deferred P→D pushes whose handshake just completed; do xfer now
|
||||
while True:
|
||||
try:
|
||||
rid, blocks, rd = self._deferred_push_inbox.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._do_start_push_kv(rid, blocks, rd)
|
||||
|
||||
# 3. P-side finished blocks; match against pending regs.
|
||||
while True:
|
||||
try:
|
||||
rid, blocks = self._finished_blocks_inbox.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
matched = self._pop_matching_registration(rid)
|
||||
if matched is not None:
|
||||
self._do_start_push_kv(rid, blocks, matched)
|
||||
else:
|
||||
self._push_finished_blocks[rid] = blocks
|
||||
|
||||
# 3b. Evict finished blocks for requests that have either
|
||||
# completed (WRITE acknowledged) or whose lease expired
|
||||
# without a D registration. Drop pending registrations
|
||||
# for the same reason so we don't leak state.
|
||||
while True:
|
||||
try:
|
||||
rid = self._evict_finished_inbox.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._push_finished_blocks.pop(rid, None)
|
||||
self._pending_d_registrations.pop(rid, None)
|
||||
|
||||
# 4. NIXL notifs: route PUSH_REG; forward the rest.
|
||||
for notifs in self.nixl_wrapper.get_new_notifs().values():
|
||||
for notif in notifs:
|
||||
if notif.startswith(PUSH_REG_NOTIF_PREFIX):
|
||||
self._handle_push_reg_notif(notif)
|
||||
else:
|
||||
self._pending_completion_notifs.put(notif)
|
||||
except Exception:
|
||||
logger.exception("nixl-push-writer error; continuing")
|
||||
|
||||
# Self-poll only while there is no other wake source: P-side
|
||||
# finished blocks waiting for a D PUSH_REG match. All other
|
||||
# progress is event-driven (see module docstring).
|
||||
if self._push_finished_blocks:
|
||||
self._push_writer_stop.wait(timeout=sleep_s)
|
||||
else:
|
||||
self._push_writer_wake.wait()
|
||||
self._push_writer_wake.clear()
|
||||
|
||||
def _handle_push_reg_notif(self, notif: bytes) -> None:
|
||||
try:
|
||||
reg_data = msgspec.msgpack.decode(notif[len(PUSH_REG_NOTIF_PREFIX) :])
|
||||
except Exception:
|
||||
logger.exception("Failed to decode PUSH_REG notification payload")
|
||||
return
|
||||
rid = reg_data.get("request_id") if isinstance(reg_data, dict) else None
|
||||
if not isinstance(rid, str):
|
||||
logger.warning("PUSH_REG notif missing request_id; dropping")
|
||||
return
|
||||
|
||||
match = self._pop_matching_finished_blocks(rid)
|
||||
if match is not None:
|
||||
fin_id, blocks = match
|
||||
self._do_start_push_kv(fin_id, blocks, reg_data)
|
||||
else:
|
||||
self._pending_d_registrations[rid] = reg_data
|
||||
|
||||
# --- D-side registration send (writer thread) ---------------------- #
|
||||
|
||||
def _send_registration_to_p(
|
||||
self,
|
||||
req_id: str,
|
||||
reg_data: dict[str, Any],
|
||||
) -> None:
|
||||
"""Handshake (if needed) then send PUSH_REG. ``send_notif`` always
|
||||
executes on the writer; the handshake runs on the background executor
|
||||
and the request is re-queued onto ``_reg_send_inbox`` once it
|
||||
completes (at which point ``_ensure_handshake`` returns ``None`` and we
|
||||
send directly)."""
|
||||
remote_pp_size = reg_data.get("remote_pp_size", 1)
|
||||
fut = self._ensure_handshake(
|
||||
reg_data["remote_engine_id"],
|
||||
reg_data["remote_host"],
|
||||
reg_data["remote_port"],
|
||||
reg_data["remote_tp_size"],
|
||||
pp_size=remote_pp_size,
|
||||
# D only ever sends PUSH_REG notifs to P and never reads or writes
|
||||
# P's memory in push mode, so it never needs the transfer
|
||||
# descriptors set up by the full add_remote_agent path.
|
||||
notif_agents_only=True,
|
||||
)
|
||||
if fut is None:
|
||||
self._do_send_reg_notif(req_id, reg_data)
|
||||
return
|
||||
|
||||
def _on_handshake(
|
||||
f: Future[tuple[dict[tuple[int, int], str], float]],
|
||||
rid: str = req_id,
|
||||
rd: dict[str, Any] = reg_data,
|
||||
) -> None:
|
||||
try:
|
||||
f.result()
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="push_reg_handshake_failed", req_id=rid, error=e
|
||||
)
|
||||
self._failed_recv_reqs.put(rid)
|
||||
return
|
||||
# Re-queue for the writer to send now that the handshake is done.
|
||||
self._reg_send_inbox.put((rid, rd))
|
||||
# Wake the writer so it sends the PUSH_REG promptly even if
|
||||
# otherwise parked.
|
||||
self._push_writer_wake.set()
|
||||
|
||||
fut.add_done_callback(_on_handshake)
|
||||
|
||||
def _do_send_reg_notif(self, req_id: str, reg_data: dict[str, Any]) -> None:
|
||||
engine_id = reg_data["remote_engine_id"]
|
||||
notif_msg = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(reg_data)
|
||||
# _remote_agents is mutated on other threads; snapshot under the lock.
|
||||
with self._handshake_lock:
|
||||
agents = dict(self._remote_agents.get(engine_id) or {})
|
||||
if not agents:
|
||||
logger.error(
|
||||
"No remote agents for engine %s; cannot send registration for %s",
|
||||
engine_id,
|
||||
req_id,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
return
|
||||
for rank, agent_name in agents.items():
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_msg)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="push_reg_notif_failed",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
remote_rank=rank,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
# Earlier registrations may still trigger WRITEs into D's blocks.
|
||||
# Keep the receive pending until those writes are finished.
|
||||
logger.debug(
|
||||
"Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg)
|
||||
)
|
||||
|
||||
# --- Matching helpers --------------------------------------------- #
|
||||
|
||||
def _pop_matching_registration(self, request_id: str) -> dict[str, Any] | None:
|
||||
"""Pop the D-side registration matching *request_id*.
|
||||
|
||||
Exact key first, then a match after stripping the random suffix from
|
||||
both sides. No match leaves the request unmatched (push not started).
|
||||
"""
|
||||
data = self._pending_d_registrations.pop(request_id, None)
|
||||
if data is not None:
|
||||
return data
|
||||
base_id = get_base_request_id(request_id)
|
||||
for reg_id in list(self._pending_d_registrations):
|
||||
if get_base_request_id(reg_id) == base_id:
|
||||
return self._pending_d_registrations.pop(reg_id)
|
||||
return None
|
||||
|
||||
def _pop_matching_finished_blocks(
|
||||
self, request_id: str
|
||||
) -> tuple[str, BlockIds] | None:
|
||||
"""Pop the P-side finished blocks matching *request_id*.
|
||||
|
||||
Same lookup as ``_pop_matching_registration``: exact key, then a
|
||||
match after stripping the random suffix from both sides.
|
||||
"""
|
||||
blocks = self._push_finished_blocks.pop(request_id, None)
|
||||
if blocks is not None:
|
||||
return request_id, blocks
|
||||
base_id = get_base_request_id(request_id)
|
||||
for fin_id in list(self._push_finished_blocks):
|
||||
if get_base_request_id(fin_id) == base_id:
|
||||
return fin_id, self._push_finished_blocks.pop(fin_id)
|
||||
return None
|
||||
|
||||
# --- WRITE transfer logic (writer thread) ------------------------- #
|
||||
|
||||
def _do_start_push_kv(
|
||||
self,
|
||||
request_id: str,
|
||||
local_block_ids: BlockIds,
|
||||
registration_data: dict[str, Any],
|
||||
) -> None:
|
||||
"""Start push-based KV transfer from P worker to D node.
|
||||
|
||||
The P→D handshake runs on the base worker's background executor.
|
||||
If it isn't ready yet we register a completion callback, defer the
|
||||
WRITE, and re-drive this request via ``_deferred_push_inbox`` once
|
||||
the handshake resolves -- so the writer thread never blocks on the
|
||||
network (mirrors ``_send_registration_to_p``).
|
||||
"""
|
||||
if not local_block_ids:
|
||||
logger.warning("No local blocks to push for request %s", request_id)
|
||||
return
|
||||
|
||||
# ``local_block_ids`` are P's logical block IDs; ``remote_block_ids``
|
||||
# (D's, from the PUSH_REG notif) are also logical.
|
||||
decode_engine_id = registration_data["decode_engine_id"]
|
||||
remote_block_ids = registration_data["local_block_ids"]
|
||||
decode_request_id = registration_data["request_id"]
|
||||
|
||||
# Runs on the background executor; defer the WRITE until it's ready.
|
||||
fut = self._ensure_handshake(
|
||||
decode_engine_id,
|
||||
registration_data["decode_host"],
|
||||
registration_data["decode_port"],
|
||||
registration_data["decode_tp_size"],
|
||||
)
|
||||
if fut is not None:
|
||||
|
||||
def _on_handshake(
|
||||
f: Future[tuple[dict[tuple[int, int], str], float]],
|
||||
rid: str = request_id,
|
||||
blocks: BlockIds = local_block_ids,
|
||||
rd: dict[str, Any] = registration_data,
|
||||
) -> None:
|
||||
if (e := f.exception()) is not None:
|
||||
# The engine reclaims the blocks via the TTL so we dont free here
|
||||
self._log_failure(
|
||||
failure_type="push_handshake_failed", req_id=rid, error=e
|
||||
)
|
||||
return
|
||||
self._deferred_push_inbox.put((rid, blocks, rd))
|
||||
self._push_writer_wake.set()
|
||||
|
||||
fut.add_done_callback(_on_handshake)
|
||||
return
|
||||
# Keep the engine alive while it is actively receiving pushes, mirroring
|
||||
# how pull-mode transfers touch _engine_last_active in start_load_kv.
|
||||
self._engine_last_active[decode_engine_id] = time.perf_counter()
|
||||
|
||||
logical_local = self._as_grouped_block_ids(local_block_ids)
|
||||
logical_remote = self._as_grouped_block_ids(remote_block_ids)
|
||||
physical_local = self._logical_to_kernel_block_ids(
|
||||
logical_local, self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
|
||||
push_meta = ReqMeta(
|
||||
local_block_ids=logical_local,
|
||||
local_physical_block_ids=physical_local,
|
||||
tp_size=self.world_size,
|
||||
remote=RemoteMeta(
|
||||
block_ids=logical_remote,
|
||||
host="",
|
||||
port=0,
|
||||
engine_id=decode_engine_id,
|
||||
request_id=decode_request_id,
|
||||
),
|
||||
)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
self._xfer_blocks_for_req(req_id=request_id, meta=push_meta)
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000.0
|
||||
if elapsed_ms > 200.0:
|
||||
logger.warning(
|
||||
"_do_start_push_kv for %s took %.1fms (slow NIXL submission)",
|
||||
request_id,
|
||||
elapsed_ms,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _as_grouped_block_ids(block_ids: BlockIds) -> BlockIds:
|
||||
"""Normalise a sequence of block IDs to a tuple-of-groups shape.
|
||||
|
||||
``BlockIds`` is canonically a tuple of per-group lists, but some
|
||||
registration payloads collapse a single-group case to a flat
|
||||
list. Re-wrap that case so downstream group-aware helpers see a
|
||||
consistent shape."""
|
||||
if block_ids and not isinstance(block_ids[0], (list, tuple)):
|
||||
return (list(block_ids),)
|
||||
return block_ids
|
||||
|
||||
def _xfer_blocks_for_req(self, req_id: str, meta: ReqMeta):
|
||||
"""Issue WRITE transfers to one or more remote TP ranks."""
|
||||
assert meta.remote is not None and self.transfer_topo is not None
|
||||
engine_id = meta.remote.engine_id
|
||||
plan = self.tp_mappings[engine_id]
|
||||
remote_info = self.transfer_topo.get_engine_info(engine_id)
|
||||
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
|
||||
|
||||
# Expand D's logical IDs using the ratio learned during the
|
||||
# NIXL handshake. ``meta`` is freshly built by
|
||||
# ``_do_start_push_kv`` so mutating it here is safe.
|
||||
meta.remote.block_ids = self._logical_to_kernel_block_ids(
|
||||
meta.remote.block_ids,
|
||||
remote_info.remote_physical_blocks_per_logical,
|
||||
)
|
||||
remote_block_ids = meta.remote.block_ids
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
local_region_groups = self.region_group_ids
|
||||
remote_region_groups = self.dst_region_group_ids[engine_id]
|
||||
groups_differ = local_region_groups != remote_region_groups
|
||||
if groups_differ and not self._transfer_layer_group_ids:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector does not support different producer and "
|
||||
"consumer cache-group layouts"
|
||||
)
|
||||
|
||||
# MLA latent is replicated across D's TP ranks: the tp-mapping
|
||||
# collapses it to one rank (fine for reads), but push must WRITE every
|
||||
# D rank or the rest decode stale KV. For hybrid MLA+SSM the sharded
|
||||
# SSM state already targets every covered D rank, so only the
|
||||
# attention groups need widening; pure MLA writes to all handshaked
|
||||
# ranks (only the dst differs per rank).
|
||||
replicate_attn = self.use_mla and tp_ratio < 0
|
||||
if replicate_attn and not self._has_mamba:
|
||||
assert len(plan.all_source_ranks) == 1
|
||||
write_ranks = sorted(self.dst_xfer_side_handles[engine_id])
|
||||
else:
|
||||
write_ranks = list(plan.all_source_ranks)
|
||||
|
||||
num_groups = len(local_block_ids)
|
||||
|
||||
def group_ids(block_ids: BlockIds, rank: int) -> BlockIds:
|
||||
return [
|
||||
list(block_ids[g])
|
||||
if (self._is_csa_linear and tp_ratio < 0)
|
||||
or (replicate_attn and _is_attention_spec(self._group_spec_types[g]))
|
||||
or rank in plan.source_ranks_per_group[g]
|
||||
else []
|
||||
for g in range(num_groups)
|
||||
]
|
||||
|
||||
read_specs = [
|
||||
ReadSpec(
|
||||
remote_rank=rank,
|
||||
local_block_ids=group_ids(local_block_ids, rank),
|
||||
remote_block_ids=group_ids(remote_block_ids, rank),
|
||||
)
|
||||
for rank in write_ranks
|
||||
]
|
||||
|
||||
handles: list[int] = []
|
||||
for i, spec in enumerate(read_specs):
|
||||
remote_block_size = remote_info.remote_block_size
|
||||
logger.debug(
|
||||
"Remote agent %s available, calling _xfer_blocks"
|
||||
" on remote rank %s with remote block size %s for req %s",
|
||||
meta.remote.engine_id,
|
||||
spec.remote_rank,
|
||||
remote_block_size,
|
||||
req_id,
|
||||
)
|
||||
if tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1):
|
||||
# Multiple targets: write each rank its chunk of local memory.
|
||||
# Hybrid MLA+SSM also lands here: its split handles replicate
|
||||
# the attention descriptors and chunk only the SSM state.
|
||||
split_key = (tp_ratio, remote_block_size)
|
||||
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
|
||||
else:
|
||||
local_xfer_side_handle = self.src_xfer_handles_by_block_size[
|
||||
remote_block_size
|
||||
]
|
||||
|
||||
remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][
|
||||
spec.remote_rank
|
||||
]
|
||||
|
||||
handle = self._xfer_blocks(
|
||||
read_spec=spec,
|
||||
request_id=req_id,
|
||||
dst_engine_id=meta.remote.engine_id,
|
||||
remote_request_id=meta.remote.request_id,
|
||||
local_xfer_side_handle=local_xfer_side_handle,
|
||||
remote_xfer_side_handle=remote_xfer_side_handle,
|
||||
)
|
||||
if handle is not None:
|
||||
handles.append(handle)
|
||||
|
||||
# Publish all the request's WRITE handles in one locked update: a
|
||||
# partial set would let ``_pop_done_transfers`` finish the request
|
||||
# early, then double-report it as the remaining writes land.
|
||||
if handles:
|
||||
with self._sending_transfers_lock:
|
||||
self._sending_transfers[req_id].extend(handles)
|
||||
|
||||
def _xfer_blocks(
|
||||
self,
|
||||
read_spec: ReadSpec,
|
||||
dst_engine_id: str,
|
||||
request_id: str,
|
||||
remote_request_id: str,
|
||||
local_xfer_side_handle: int,
|
||||
remote_xfer_side_handle: int,
|
||||
) -> int | None:
|
||||
"""Post a WRITE point-to-point xfer request.
|
||||
|
||||
Returns the in-flight transfer handle (so the caller can track all of
|
||||
a request's handles atomically), or ``None`` if nothing was submitted.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
remote_rank = read_spec.remote_rank
|
||||
local_block_ids = read_spec.local_block_ids
|
||||
remote_block_ids = read_spec.remote_block_ids
|
||||
|
||||
remote_info = self.transfer_topo.get_engine_info(dst_engine_id)
|
||||
block_size_ratio = self.transfer_topo.block_size_ratio(
|
||||
remote_info.remote_block_size
|
||||
)
|
||||
if block_size_ratio > 1:
|
||||
local_block_ids, remote_block_ids = (
|
||||
self._map_block_ids_for_block_size_ratio(
|
||||
local_block_ids, remote_block_ids, block_size_ratio
|
||||
)
|
||||
)
|
||||
|
||||
notif_id = f"{remote_request_id}:{self.world_size}".encode()
|
||||
|
||||
if len(local_block_ids) == 0:
|
||||
logger.warning("No blocks to push for request %s", request_id)
|
||||
return None
|
||||
|
||||
# Prefix caching: D allocated only uncached blocks, so on a partial hit it
|
||||
# sends fewer than P's. End-trim P's blocks to that same suffix so we WRITE only
|
||||
# the uncomputed tail into D's slots. Runs on kernel ids, post-expansion.
|
||||
remote_block_ids, local_block_ids = self._apply_prefix_caching(
|
||||
decode_block_ids=remote_block_ids,
|
||||
prefill_block_ids=local_block_ids,
|
||||
decode_physical_per_logical=remote_info.remote_physical_blocks_per_logical,
|
||||
prefill_physical_per_logical=self._physical_blocks_per_logical_kv_block,
|
||||
)
|
||||
|
||||
local_block_ids = list(local_block_ids)
|
||||
remote_block_ids = list(remote_block_ids)
|
||||
assert len(local_block_ids) == len(remote_block_ids), (
|
||||
f"push group-count mismatch for {request_id}: {len(local_block_ids)} "
|
||||
f"local vs {len(remote_block_ids)} remote groups"
|
||||
)
|
||||
for i in range(len(local_block_ids)):
|
||||
assert len(local_block_ids[i]) == len(remote_block_ids[i]), (
|
||||
f"push block-count mismatch for {request_id} group {i}: "
|
||||
f"{len(local_block_ids[i])} local vs "
|
||||
f"{len(remote_block_ids[i])} remote blocks"
|
||||
)
|
||||
|
||||
# Get descs ids.
|
||||
remote_block_descs_ids = self._compute_desc_ids(
|
||||
block_ids=remote_block_ids,
|
||||
dst_num_blocks=self.dst_num_blocks[dst_engine_id],
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical,
|
||||
region_num_blocks=self.dst_region_num_blocks[dst_engine_id],
|
||||
region_group_ids=self.dst_region_group_ids[dst_engine_id],
|
||||
uses_region_group_mapping=self.dst_uses_region_group_mapping[dst_engine_id],
|
||||
)
|
||||
local_block_descs_ids = self._compute_desc_ids(
|
||||
block_ids=local_block_ids,
|
||||
dst_num_blocks=self.dst_num_blocks[self.engine_id],
|
||||
block_size_ratio=block_size_ratio,
|
||||
physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block,
|
||||
region_num_blocks=self.dst_region_num_blocks[self.engine_id],
|
||||
region_group_ids=self.region_group_ids,
|
||||
uses_region_group_mapping=self._uses_region_group_mapping,
|
||||
)
|
||||
|
||||
assert len(local_block_descs_ids) == len(remote_block_descs_ids)
|
||||
|
||||
handle = None
|
||||
try:
|
||||
handle = self.nixl_wrapper.make_prepped_xfer(
|
||||
"WRITE",
|
||||
local_xfer_side_handle,
|
||||
local_block_descs_ids,
|
||||
remote_xfer_side_handle,
|
||||
remote_block_descs_ids,
|
||||
notif_msg=notif_id,
|
||||
)
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
# Caller tracks the handle (atomically with the request's other
|
||||
# writes) so P can free blocks once all of them are done.
|
||||
return handle
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="transfer_setup_failed",
|
||||
req_id=request_id,
|
||||
msg="Push WRITE submission failed; releasing handle",
|
||||
error=e,
|
||||
dst_engine_id=dst_engine_id,
|
||||
remote_rank=remote_rank,
|
||||
)
|
||||
# On the P side this WRITE failure is purely outbound; we
|
||||
# don't have a ``_recving_metadata`` entry to invalidate, so
|
||||
# we just release the handle and let the engine reschedule
|
||||
# via the lease / watchdog.
|
||||
with self._sending_transfers_lock:
|
||||
self._send_failures.add(request_id)
|
||||
if not self._handle_failed_transfer(request_id, handle):
|
||||
return handle
|
||||
return None
|
||||
|
||||
# --- Notification handling on engine main thread ------------------ #
|
||||
|
||||
def _get_new_notifs(self) -> set[str]:
|
||||
"""Drain HB / completion notifs forwarded by the writer thread.
|
||||
|
||||
The writer owns ``nixl_wrapper.get_new_notifs`` for push; PUSH_REG
|
||||
notifs are handled there. P learns of finished pushes from its WRITE
|
||||
handles, so this never reports finished sends.
|
||||
"""
|
||||
while True:
|
||||
try:
|
||||
notif = self._pending_completion_notifs.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
msg = notif.decode("utf-8")
|
||||
if msg.startswith("HB:"):
|
||||
self._handle_heartbeat(msg[3:])
|
||||
continue
|
||||
|
||||
req_id, tp_size = msg.rsplit(":", 1)
|
||||
|
||||
if (meta := self._recving_metadata.get(req_id)) is None:
|
||||
# The lease may have expired before the notif arrived.
|
||||
self.xfer_stats.record_notification_after_expiry()
|
||||
logger.error(
|
||||
"Unrecognized request %s notif (may have expired).", req_id
|
||||
)
|
||||
continue
|
||||
# Consumer waits for one notif per producer rank writing
|
||||
# here: pp_size stages * producers-per-consumer (>1 when
|
||||
# producer TP > consumer TP; tp_size is the producer TP).
|
||||
producers_per_consumer = max(1, int(tp_size) // self.world_size)
|
||||
expected_notifs = meta.pp_size * producers_per_consumer
|
||||
self.consumer_notification_counts_by_req[req_id] += 1
|
||||
if self.consumer_notification_counts_by_req[req_id] < expected_notifs:
|
||||
continue
|
||||
del self.consumer_notification_counts_by_req[req_id]
|
||||
# P drove the transfer (we own no NIXL handle), so
|
||||
# materialise an empty ``_recving_transfers`` entry for
|
||||
# ``_pop_done_transfers`` to report done.
|
||||
self._recving_transfers.setdefault(req_id, [])
|
||||
return set()
|
||||
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
# Engine main thread asking for completions: also wake the writer
|
||||
# so it gets a chance to drain NIXL notifs (heartbeats, completion
|
||||
# notifs, late PUSH_REGs) even if it had been parked.
|
||||
self._push_writer_wake.set()
|
||||
|
||||
results = super().get_transfer_results()
|
||||
done_sending = results.finished_sending
|
||||
|
||||
# ``_pop_done_transfers`` mutates ``_sending_transfers``; the
|
||||
# writer thread also appends to it, so guard the pop.
|
||||
with self._sending_transfers_lock:
|
||||
done_pushing, failed_pushing = self._pop_done_transfers(
|
||||
self._sending_transfers
|
||||
)
|
||||
# Remember failures until the final sibling WRITE completes.
|
||||
self._send_failures.update(failed_pushing)
|
||||
successful = {
|
||||
req_id
|
||||
for req_id in done_pushing - self._send_failures
|
||||
if req_id in self._reqs_to_send or req_id in self._reqs_to_process
|
||||
}
|
||||
self._send_failures.difference_update(done_pushing | done_sending)
|
||||
# Expired requests were already reported, even if their WRITEs finish later.
|
||||
done_pushing = successful
|
||||
for req_id in done_pushing:
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
self._reqs_to_process.discard(req_id)
|
||||
done_sending.add(req_id)
|
||||
|
||||
# Tell the writer to drop any state it still holds for any
|
||||
# request that just finished (push completed) or expired
|
||||
# (lease ran out without a D registration ever arriving).
|
||||
for req_id in done_sending:
|
||||
self._evict_finished_inbox.put(req_id)
|
||||
if done_sending:
|
||||
self._push_writer_wake.set()
|
||||
|
||||
return results
|
||||
@@ -1,12 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backward-compatible re-export of NixlPullConnectorScheduler."""
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
|
||||
# Backward compatibility: NixlConnectorScheduler is the pull-based scheduler.
|
||||
NixlConnectorScheduler = NixlPullConnectorScheduler
|
||||
|
||||
__all__ = ["NixlConnectorScheduler", "NixlPullConnectorScheduler"]
|
||||
@@ -1,328 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stats and Prometheus metrics for the NIXL connector."""
|
||||
|
||||
import copy
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.v1.metrics.utils import create_metric_per_engine
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.nixl_utils import nixlXferTelemetry
|
||||
|
||||
|
||||
@dataclass
|
||||
class NixlKVConnectorStats(KVConnectorStats):
|
||||
"""Container for transfer performance metrics."""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
# Empty container init, no data is passed in.
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
# Must be serializable
|
||||
self.data: dict[str, list[float | int]] = {
|
||||
"transfer_duration": [],
|
||||
"post_duration": [],
|
||||
"bytes_transferred": [],
|
||||
"num_descriptors": [],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
|
||||
def record_transfer(self, res: "nixlXferTelemetry | None"):
|
||||
# Keep metrics units consistent with rest of the code: time us->s
|
||||
if res is None:
|
||||
return
|
||||
self.data["transfer_duration"].append(res.xferDuration / 1e6)
|
||||
self.data["post_duration"].append(res.postDuration / 1e6)
|
||||
self.data["bytes_transferred"].append(res.totalBytes)
|
||||
self.data["num_descriptors"].append(res.descCount)
|
||||
|
||||
def record_failed_transfer(self):
|
||||
"""Record a failed NIXL transfer operation."""
|
||||
self.data["num_failed_transfers"].append(1)
|
||||
|
||||
def record_failed_notification(self):
|
||||
"""Record a failed NIXL notification (send_notif)."""
|
||||
self.data["num_failed_notifications"].append(1)
|
||||
|
||||
def record_failed_handshake(self):
|
||||
"""Record a failed NIXL handshake."""
|
||||
self.data["num_failed_handshakes"].append(1)
|
||||
|
||||
def record_kv_expired_req(self):
|
||||
"""Record a request that had its KV blocks expire."""
|
||||
self.data["num_kv_expired_reqs"].append(1)
|
||||
|
||||
def record_notification_after_expiry(self):
|
||||
"""Record a completion notification that arrived after its request expired."""
|
||||
self.data["num_notifications_after_expiry"].append(1)
|
||||
|
||||
def clone_and_reset(self) -> "NixlKVConnectorStats":
|
||||
old = copy.copy(self)
|
||||
self.reset()
|
||||
return old
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
# Do not discard metrics update that are entirely failures related.
|
||||
return (
|
||||
self.num_successful_transfers == 0
|
||||
and len(self.data["num_failed_transfers"]) == 0
|
||||
and len(self.data["num_failed_notifications"]) == 0
|
||||
and len(self.data["num_failed_handshakes"]) == 0
|
||||
and len(self.data["num_kv_expired_reqs"]) == 0
|
||||
and len(self.data["num_notifications_after_expiry"]) == 0
|
||||
)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
accumulator.extend(v)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
# Compute compact representative stats suitable for CLI logging.
|
||||
# Failure counts are reported on every interval: transfer, handshake
|
||||
# and notification failures are grouped as sporadic
|
||||
# lower-transport-layer events, while KV expiry is reported separately
|
||||
# as it is an actionable autoscaler signal rather than a transport
|
||||
# issue. Notifications arriving after expiry are surfaced too, as the
|
||||
# reader may have consumed KV blocks that were already reused.
|
||||
failure_counts = {
|
||||
"Num failed transfers": len(self.data["num_failed_transfers"])
|
||||
+ len(self.data["num_failed_handshakes"])
|
||||
+ len(self.data["num_failed_notifications"]),
|
||||
"Num KV expired reqs": len(self.data["num_kv_expired_reqs"]),
|
||||
"Num notifs after expiry": len(self.data["num_notifications_after_expiry"]),
|
||||
}
|
||||
if self.num_successful_transfers == 0:
|
||||
# Timing / throughput stats only cover successful transfers. If
|
||||
# all requests in the interval were unsuccessful, the failure
|
||||
# counts above still surface what happened.
|
||||
return {
|
||||
"Num successful transfers": 0,
|
||||
"Avg xfer time (ms)": 0,
|
||||
"P90 xfer time (ms)": 0,
|
||||
"Avg post time (ms)": 0,
|
||||
"P90 post time (ms)": 0,
|
||||
"Avg MB per transfer": 0,
|
||||
"Throughput (MB/s)": 0,
|
||||
"Avg number of descriptors": 0,
|
||||
**failure_counts,
|
||||
}
|
||||
|
||||
xfer_time = np.asarray(self.data["transfer_duration"])
|
||||
post_time = np.asarray(self.data["post_duration"])
|
||||
# Convert to MB for CLI logging.
|
||||
mb = np.asarray(self.data["bytes_transferred"]) / 2**20
|
||||
descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32)
|
||||
n = len(descs)
|
||||
assert n == self.num_successful_transfers
|
||||
|
||||
total_mb = mb.sum()
|
||||
avg_mb = total_mb / n
|
||||
|
||||
total_time_seconds = xfer_time.sum()
|
||||
throughput_mb_s = total_mb / total_time_seconds
|
||||
|
||||
return {
|
||||
"Num successful transfers": n,
|
||||
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
|
||||
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
|
||||
"Avg post time (ms)": round(post_time.mean().item() * 1e3, 3),
|
||||
"P90 post time (ms)": round(np.percentile(post_time, 90).item() * 1e3, 3),
|
||||
"Avg MB per transfer": round(avg_mb.item(), 3),
|
||||
"Throughput (MB/s)": round(throughput_mb_s.item(), 3),
|
||||
"Avg number of descriptors": round(descs.mean().item(), 1),
|
||||
**failure_counts,
|
||||
}
|
||||
|
||||
@property
|
||||
def num_successful_transfers(self) -> int:
|
||||
return len(self.data["transfer_duration"])
|
||||
|
||||
|
||||
class NixlPromMetrics(KVConnectorPromMetrics):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
|
||||
buckets = [
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.025,
|
||||
0.05,
|
||||
0.075,
|
||||
0.1,
|
||||
0.2,
|
||||
0.3,
|
||||
0.5,
|
||||
0.75,
|
||||
1.0,
|
||||
5.0,
|
||||
]
|
||||
nixl_histogram_xfer_time = self._histogram_cls(
|
||||
name="vllm:nixl_xfer_time_seconds",
|
||||
documentation="Histogram of transfer duration for NIXL KV Cache transfers.",
|
||||
buckets=buckets[1:],
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.nixl_histogram_xfer_time = create_metric_per_engine(
|
||||
nixl_histogram_xfer_time, self.per_engine_labelvalues
|
||||
)
|
||||
nixl_histogram_post_time = self._histogram_cls(
|
||||
name="vllm:nixl_post_time_seconds",
|
||||
documentation="Histogram of transfer post time for NIXL KV"
|
||||
" Cache transfers.",
|
||||
buckets=buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.nixl_histogram_post_time = create_metric_per_engine(
|
||||
nixl_histogram_post_time, self.per_engine_labelvalues
|
||||
)
|
||||
# uniform 2kb to 16gb range
|
||||
buckets = [2 ** (10 + i) for i in range(1, 25, 2)]
|
||||
nixl_histogram_bytes_transferred = self._histogram_cls(
|
||||
name="vllm:nixl_bytes_transferred",
|
||||
documentation="Histogram of bytes transferred per NIXL KV Cache transfers.",
|
||||
buckets=buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.nixl_histogram_bytes_transferred = create_metric_per_engine(
|
||||
nixl_histogram_bytes_transferred, self.per_engine_labelvalues
|
||||
)
|
||||
buckets = [
|
||||
10,
|
||||
20,
|
||||
30,
|
||||
50,
|
||||
75,
|
||||
100,
|
||||
200,
|
||||
400,
|
||||
1000,
|
||||
2000,
|
||||
4000,
|
||||
10000,
|
||||
20000,
|
||||
50000,
|
||||
]
|
||||
nixl_histogram_num_descriptors = self._histogram_cls(
|
||||
name="vllm:nixl_num_descriptors",
|
||||
documentation="Histogram of number of descriptors per NIXL"
|
||||
" KV Cache transfers.",
|
||||
buckets=buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.nixl_histogram_num_descriptors = create_metric_per_engine(
|
||||
nixl_histogram_num_descriptors, self.per_engine_labelvalues
|
||||
)
|
||||
counter_nixl_num_failed_transfers = self._counter_cls(
|
||||
name="vllm:nixl_num_failed_transfers",
|
||||
documentation="Number of failed NIXL KV Cache transfers, including"
|
||||
" handshake and notification failures. NOTE: KV expiry is tracked"
|
||||
" separately in vllm:nixl_num_kv_expired_reqs.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_failed_transfers = create_metric_per_engine(
|
||||
counter_nixl_num_failed_transfers, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_nixl_num_failed_notifications = self._counter_cls(
|
||||
name="vllm:nixl_num_failed_notifications",
|
||||
documentation="Number of failed NIXL KV Cache notifications. "
|
||||
"Retained for compatibility; these failures are also included in "
|
||||
"vllm:nixl_num_failed_transfers.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_failed_notifications = create_metric_per_engine(
|
||||
counter_nixl_num_failed_notifications, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_nixl_num_kv_expired_reqs = self._counter_cls(
|
||||
name="vllm:nixl_num_kv_expired_reqs",
|
||||
documentation="Number of requests that had their KV expire. "
|
||||
"NOTE: This metric is tracked on the P instance.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_kv_expired_reqs = create_metric_per_engine(
|
||||
counter_nixl_num_kv_expired_reqs, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_nixl_num_notifications_after_expiry = self._counter_cls(
|
||||
name="vllm:nixl_num_notifications_after_expiry",
|
||||
documentation="Number of completion notifications for requests that "
|
||||
"were no longer tracked, usually because their KV lease expired. The "
|
||||
"KV blocks may have been reused before the transfer finished. Counted "
|
||||
"per notification (one per remote rank), not per request.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_notifications_after_expiry = create_metric_per_engine(
|
||||
counter_nixl_num_notifications_after_expiry, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
for prom_obj, list_item_key in zip(
|
||||
[
|
||||
self.nixl_histogram_xfer_time,
|
||||
self.nixl_histogram_post_time,
|
||||
self.nixl_histogram_bytes_transferred,
|
||||
self.nixl_histogram_num_descriptors,
|
||||
],
|
||||
[
|
||||
"transfer_duration",
|
||||
"post_duration",
|
||||
"bytes_transferred",
|
||||
"num_descriptors",
|
||||
],
|
||||
):
|
||||
for list_item in transfer_stats_data[list_item_key]:
|
||||
prom_obj[engine_idx].observe(list_item)
|
||||
for counter_obj, counter_item_keys in zip(
|
||||
[
|
||||
self.counter_nixl_num_failed_transfers,
|
||||
self.counter_nixl_num_failed_notifications,
|
||||
self.counter_nixl_num_kv_expired_reqs,
|
||||
self.counter_nixl_num_notifications_after_expiry,
|
||||
],
|
||||
[
|
||||
# Transfer, handshake and notification failures are grouped:
|
||||
# all are sporadic lower-transport-layer events. KV expiry is
|
||||
# reported separately as it signals autoscaler behavior, not
|
||||
# transport health.
|
||||
(
|
||||
"num_failed_transfers",
|
||||
"num_failed_handshakes",
|
||||
"num_failed_notifications",
|
||||
),
|
||||
("num_failed_notifications",),
|
||||
("num_kv_expired_reqs",),
|
||||
("num_notifications_after_expiry",),
|
||||
],
|
||||
):
|
||||
for counter_item_key in counter_item_keys:
|
||||
for list_item in transfer_stats_data[counter_item_key]:
|
||||
counter_obj[engine_idx].inc(list_item)
|
||||
@@ -1,161 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TP mapping computation for NIXL KV cache transfers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
TransferTopology,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, MambaSpec
|
||||
|
||||
# ======================================================================
|
||||
# Data structures
|
||||
# ======================================================================
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReadSpec:
|
||||
"""Specification for a single remote block read operation."""
|
||||
|
||||
remote_rank: int
|
||||
local_block_ids: BlockIds
|
||||
remote_block_ids: BlockIds
|
||||
block_ids_by_region: bool = False
|
||||
|
||||
|
||||
def _is_attention_spec(spec_type: type[KVCacheSpec]) -> bool:
|
||||
return issubclass(spec_type, AttentionSpec)
|
||||
|
||||
|
||||
def _is_ssm_spec(spec_type: type[KVCacheSpec]) -> bool:
|
||||
return issubclass(spec_type, MambaSpec)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TPMapping:
|
||||
"""Complete local-to-remote TP mapping for one remote engine.
|
||||
|
||||
Generated once per remote engine during handshake.
|
||||
"""
|
||||
|
||||
# Remote TP ranks that this local rank reads from, per group.
|
||||
# Position = local piece index.
|
||||
source_ranks_per_group: tuple[tuple[int, ...], ...]
|
||||
|
||||
# Superset of all source ranks (union of all groups).
|
||||
all_source_ranks: tuple[int, ...]
|
||||
|
||||
# Maps each source rank to its FA head slot index.
|
||||
rank_to_attention_slot: dict[int, int]
|
||||
|
||||
# FA head offset factor for hetero-TP (D_TP > P_TP).
|
||||
rank_offset_factor: int
|
||||
|
||||
# Local ranks (in aggregate) that read from a given source rank. The producer frees
|
||||
# a request's blocks only once that many notifications have come in.
|
||||
local_consumers: int = 1
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# TP mapping computation
|
||||
# ======================================================================
|
||||
|
||||
|
||||
def compute_tp_mapping(
|
||||
transfer_topology: TransferTopology,
|
||||
remote_tp_size: int,
|
||||
group_spec_types: tuple[type[KVCacheSpec], ...],
|
||||
remote_dcp_size: int = 1,
|
||||
) -> TPMapping:
|
||||
"""Build the complete local-to-remote TP mapping.
|
||||
|
||||
Computes source ranks, head slot assignments, and the rank offset
|
||||
factor in a single pass.
|
||||
|
||||
DCP support is scoped to MLA only, with a side is either fully replicated or fully
|
||||
sharded. DCP-branch reuses the same rank set used at handshake selection.
|
||||
"""
|
||||
tp_rank = transfer_topology.tp_rank
|
||||
tp_size = transfer_topology.tp_size
|
||||
total_num_kv_heads = transfer_topology.total_num_kv_heads
|
||||
# --- Attention source ranks ---
|
||||
if transfer_topology.is_mla or tp_size >= remote_tp_size:
|
||||
if transfer_topology.is_mla and remote_dcp_size > 1:
|
||||
attn_ranks = transfer_topology.dcp_source_ranks(
|
||||
remote_tp_size, remote_dcp_size
|
||||
)
|
||||
else:
|
||||
# D (local TP) > P (remote TP): multiple local ranks read different chunks
|
||||
# from *one* remote rank, corresponding to different kv heads.
|
||||
# For MLA, we only need one remote since cache is duplicated. When
|
||||
# P TP=k*TP k, this will spread mla ranks to read from remote k*tp_rank.
|
||||
attn_ranks = [tp_rank * remote_tp_size // tp_size]
|
||||
else:
|
||||
# P (remote TP) > D (local TP): one local rank
|
||||
# reads from multiple remote ranks.
|
||||
# GQA dedup: when K < remote_tp_size, several remote ranks
|
||||
# hold the same KV head. np.unique keeps only the first
|
||||
# rank per unique head so we don't issue redundant reads.
|
||||
abs_tp = remote_tp_size // tp_size
|
||||
start = tp_rank * abs_tp
|
||||
heads = np.arange(start, start + abs_tp) * total_num_kv_heads // remote_tp_size
|
||||
_, unique_idx = np.unique(heads, return_index=True)
|
||||
attn_ranks = (start + np.sort(unique_idx)).tolist()
|
||||
|
||||
# --- SSM source ranks ---
|
||||
has_ssm = any(_is_ssm_spec(t) for t in group_spec_types)
|
||||
if has_ssm:
|
||||
if tp_size < remote_tp_size:
|
||||
abs_tp = remote_tp_size // tp_size
|
||||
ssm_ranks = list(range(tp_rank * abs_tp, (tp_rank + 1) * abs_tp))
|
||||
else:
|
||||
ssm_ranks = list(attn_ranks)
|
||||
else:
|
||||
ssm_ranks = []
|
||||
|
||||
all_ranks = sorted(set(attn_ranks) | set(ssm_ranks))
|
||||
|
||||
# --- Per-group ordered source ranks ---
|
||||
source_ranks_per_group = tuple(
|
||||
tuple(ssm_ranks) if _is_ssm_spec(t) else tuple(attn_ranks)
|
||||
for t in group_spec_types
|
||||
)
|
||||
|
||||
# --- Attention head slots ---
|
||||
head_to_slot: dict[int, int] = {}
|
||||
for i, r in enumerate(attn_ranks):
|
||||
head_to_slot[r * total_num_kv_heads // remote_tp_size] = i
|
||||
rank_to_attention_slot = {
|
||||
r: head_to_slot.get(r * total_num_kv_heads // remote_tp_size, 0)
|
||||
for r in all_ranks
|
||||
}
|
||||
|
||||
# --- Rank offset factor ---
|
||||
if transfer_topology.is_mla or tp_size <= remote_tp_size:
|
||||
# We don't index into remote for reading, no offset needed.
|
||||
rank_offset_factor = 0
|
||||
elif tp_size > total_num_kv_heads:
|
||||
local_head = tp_rank * total_num_kv_heads // tp_size
|
||||
p_start = attn_ranks[0] * total_num_kv_heads // remote_tp_size
|
||||
rank_offset_factor = local_head - p_start
|
||||
else:
|
||||
# D TP > P TP: we index into remote to read different heads depending on rank.
|
||||
rank_offset_factor = tp_rank % (tp_size // remote_tp_size)
|
||||
|
||||
local_consumers = transfer_topology.dcp_consumer_count(
|
||||
remote_tp_size, remote_dcp_size
|
||||
)
|
||||
|
||||
return TPMapping(
|
||||
source_ranks_per_group=source_ranks_per_group,
|
||||
all_source_ranks=tuple(all_ranks),
|
||||
rank_to_attention_slot=rank_to_attention_slot,
|
||||
rank_offset_factor=rank_offset_factor,
|
||||
local_consumers=local_consumers,
|
||||
)
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared constants, lazy imports and helpers for the NIXL connector."""
|
||||
|
||||
import contextlib
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import zmq
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import make_zmq_socket
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, UniformTypeKVCacheSpecs
|
||||
|
||||
# Supported platforms and types of kv transfer buffer.
|
||||
# {device: tuple of supported kv buffer types}
|
||||
_NIXL_SUPPORTED_DEVICE = {
|
||||
"cuda": (
|
||||
"cuda",
|
||||
"cpu",
|
||||
),
|
||||
"tpu": ("cpu",),
|
||||
"xpu": (
|
||||
"cpu",
|
||||
"xpu",
|
||||
),
|
||||
"cpu": ("cpu",),
|
||||
}
|
||||
# support for oot platform by providing mapping in current_platform
|
||||
_NIXL_SUPPORTED_DEVICE.update(current_platform.get_nixl_supported_devices())
|
||||
|
||||
|
||||
# TODO: merge with vllm.utils.network_utils.zmq_socket_ctx
|
||||
@contextlib.contextmanager
|
||||
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
|
||||
"""Context manager for a ZMQ socket."""
|
||||
if socket_type not in (zmq.ROUTER, zmq.REQ):
|
||||
raise ValueError(f"Unexpected socket type: {socket_type}")
|
||||
|
||||
ctx: zmq.Context | None = None
|
||||
try:
|
||||
ctx = zmq.Context() # type: ignore[attr-defined]
|
||||
yield make_zmq_socket(
|
||||
ctx=ctx, path=addr, socket_type=socket_type, bind=socket_type == zmq.ROUTER
|
||||
)
|
||||
finally:
|
||||
if ctx is not None:
|
||||
ctx.destroy(linger=0)
|
||||
|
||||
|
||||
def get_representative_spec_type(spec: KVCacheSpec) -> type[KVCacheSpec]:
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
# All inner specs are the same type; pick any.
|
||||
inner = next(iter(spec.kv_cache_specs.values()))
|
||||
return type(inner)
|
||||
return type(spec)
|
||||
|
||||
|
||||
# Trailing 8-hex randomization suffix appended by
|
||||
# ``input_processor.assign_request_id`` as ``-{random_uuid():.8}``.
|
||||
_RANDOM_SUFFIX_RE = re.compile(r"-[0-9a-f]{8}$", re.IGNORECASE)
|
||||
|
||||
|
||||
def get_base_request_id(request_id: str) -> str:
|
||||
"""Strip the per-request ``-<8 hex>`` randomization suffix, if present."""
|
||||
return _RANDOM_SUFFIX_RE.sub("", request_id)
|
||||
@@ -1,13 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backward-compatible re-export of NixlPullConnectorWorker."""
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import (
|
||||
NixlPullConnectorWorker,
|
||||
)
|
||||
|
||||
# Backward compatibility: NixlConnectorWorker is the pull-based worker.
|
||||
NixlConnectorWorker = NixlPullConnectorWorker
|
||||
|
||||
|
||||
__all__ = ["NixlConnectorWorker", "NixlPullConnectorWorker"]
|
||||
@@ -1,386 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""SimpleCPUOffloadConnector: minimal CPU KV cache offloading."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_events import KVCacheEvent
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.simple_kv_offload.manager import (
|
||||
BoundaryStoreStats,
|
||||
SimpleCPUOffloadScheduler,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metadata import (
|
||||
SimpleCPUOffloadMetadata,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
SimpleCPUOffloadPromMetrics,
|
||||
SimpleCPUOffloadStats,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.worker import (
|
||||
SimpleCPUOffloadWorker,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Default CPU capacity: 8 GB
|
||||
DEFAULT_CPU_CAPACITY_BYTES = 8 * (1024**3)
|
||||
|
||||
VALID_KV_OFFLOAD_BACKENDS = ("cpu", "disk")
|
||||
# Keys that only apply to the disk backend, warned about under "cpu".
|
||||
_DISK_ONLY_KEYS = (
|
||||
"disk_path",
|
||||
"disk_capacity_bytes",
|
||||
"disk_buffer_slots",
|
||||
"use_page_cache",
|
||||
)
|
||||
|
||||
|
||||
class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""CPU KV cache offloading with custom kernel transfers and BlockPool LRU."""
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
|
||||
# future cache miss, so opt out of the producer-role default.
|
||||
return False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
|
||||
enable_prefix_caching = vllm_config.cache_config.enable_prefix_caching
|
||||
extra_config = self._kv_transfer_config.kv_connector_extra_config or {}
|
||||
|
||||
cpu_capacity_bytes = int(
|
||||
extra_config.get("cpu_bytes_to_use", DEFAULT_CPU_CAPACITY_BYTES)
|
||||
)
|
||||
# cpu_bytes_to_use is server-wide for compatibility;
|
||||
# cpu_bytes_to_use_per_rank overrides for per-rank capacity.
|
||||
world_size = vllm_config.parallel_config.world_size
|
||||
cpu_capacity_per_rank = cpu_capacity_bytes // world_size
|
||||
if "cpu_bytes_to_use_per_rank" in extra_config:
|
||||
explicit = int(extra_config["cpu_bytes_to_use_per_rank"])
|
||||
if explicit != cpu_capacity_per_rank:
|
||||
logger.warning(
|
||||
"cpu_bytes_to_use_per_rank (%.2f GB) != "
|
||||
"cpu_bytes_to_use/world_size (%.2f GB). Using per-rank value.",
|
||||
explicit / (1024**3),
|
||||
cpu_capacity_per_rank / (1024**3),
|
||||
)
|
||||
cpu_capacity_per_rank = explicit
|
||||
|
||||
# When set, num_cpu_blocks overrides automatic pool sizing from byte budgets
|
||||
# (including cpu_bytes_to_use and cpu_bytes_to_use_per_rank), allowing ranks
|
||||
# with different KV layouts (e.g. PP stages) to use the same pool size.
|
||||
# Byte budgets still bound capacity.
|
||||
num_cpu_blocks = extra_config.get("num_cpu_blocks")
|
||||
if num_cpu_blocks is not None and (
|
||||
type(num_cpu_blocks) is not int or num_cpu_blocks <= 0
|
||||
):
|
||||
raise ValueError("num_cpu_blocks must be a positive integer")
|
||||
|
||||
lazy_offload = bool(extra_config.get("lazy_offload", False))
|
||||
|
||||
kv_offload_backend = str(extra_config.get("kv_offload_backend", "cpu"))
|
||||
if kv_offload_backend not in VALID_KV_OFFLOAD_BACKENDS:
|
||||
raise ValueError(
|
||||
f"Unknown kv_offload_backend {kv_offload_backend!r}; "
|
||||
f"expected one of {VALID_KV_OFFLOAD_BACKENDS}"
|
||||
)
|
||||
disk_mode = kv_offload_backend == "disk"
|
||||
self._cache_hit_source = (
|
||||
CacheHitSource.DISK if disk_mode else CacheHitSource.HOST
|
||||
)
|
||||
|
||||
disk_path = extra_config.get("disk_path", None) or None
|
||||
disk_capacity_bytes = int(
|
||||
extra_config.get("disk_capacity_bytes", 100 * (1024**3))
|
||||
)
|
||||
disk_buffer_slots = max(1, int(extra_config.get("disk_buffer_slots", 2)))
|
||||
use_page_cache = bool(extra_config.get("use_page_cache", False))
|
||||
|
||||
if disk_mode:
|
||||
if disk_path is None:
|
||||
raise ValueError(
|
||||
'kv_offload_backend="disk" requires disk_path to be set.'
|
||||
)
|
||||
if disk_capacity_bytes <= 0:
|
||||
raise ValueError(
|
||||
'kv_offload_backend="disk" requires disk_capacity_bytes > 0.'
|
||||
)
|
||||
else:
|
||||
ignored = [k for k in _DISK_ONLY_KEYS if k in extra_config]
|
||||
if ignored:
|
||||
logger.warning(
|
||||
'kv_offload_backend is "cpu", ignoring disk-only config: %s. '
|
||||
'Set kv_offload_backend="disk" to enable the disk backend.',
|
||||
", ".join(ignored),
|
||||
)
|
||||
disk_path = None
|
||||
|
||||
self.scheduler_manager: SimpleCPUOffloadScheduler | None = None
|
||||
self.worker_handler: SimpleCPUOffloadWorker | None = None
|
||||
|
||||
if not enable_prefix_caching:
|
||||
logger.warning(
|
||||
"Detected prefix caching disabled, disabling CPU offload "
|
||||
"since it requires prefix caching."
|
||||
)
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"SimpleCPUOffloadConnector: role=%s, "
|
||||
"per_rank=%.2f GB, world_size=%d, mode=%s, backend=%s, disk=%s",
|
||||
role.name,
|
||||
cpu_capacity_per_rank / (1024**3),
|
||||
world_size,
|
||||
"lazy" if lazy_offload else "eager",
|
||||
kv_offload_backend,
|
||||
disk_path or "none",
|
||||
)
|
||||
if num_cpu_blocks is not None:
|
||||
logger.info(
|
||||
"Using explicit num_cpu_blocks=%d for offload pool sizing "
|
||||
"instead of automatic sizing from byte budgets.",
|
||||
num_cpu_blocks,
|
||||
)
|
||||
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
|
||||
assert kv_cache_config is not None
|
||||
scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
self.scheduler_manager = SimpleCPUOffloadScheduler(
|
||||
vllm_config,
|
||||
kv_cache_config,
|
||||
cpu_capacity_per_rank,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
lazy_offload=lazy_offload,
|
||||
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
|
||||
use_page_cache=use_page_cache if disk_mode else False,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.worker_handler = SimpleCPUOffloadWorker(
|
||||
vllm_config,
|
||||
kv_cache_config,
|
||||
cpu_capacity_per_rank,
|
||||
kv_offload_backend=kv_offload_backend,
|
||||
disk_path=disk_path,
|
||||
disk_capacity_bytes=disk_capacity_bytes,
|
||||
disk_buffer_slots=disk_buffer_slots,
|
||||
use_page_cache=use_page_cache,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
|
||||
# --- Worker-side methods ---
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
if self.worker_handler is not None:
|
||||
self.worker_handler.register_kv_caches(kv_caches)
|
||||
|
||||
def bind_connector_metadata(
|
||||
self,
|
||||
connector_metadata: KVConnectorMetadata,
|
||||
) -> None:
|
||||
super().bind_connector_metadata(connector_metadata)
|
||||
if self.worker_handler is not None:
|
||||
assert isinstance(connector_metadata, SimpleCPUOffloadMetadata)
|
||||
self.worker_handler.bind_connector_metadata(connector_metadata)
|
||||
|
||||
def clear_connector_metadata(self) -> None:
|
||||
super().clear_connector_metadata()
|
||||
if self.worker_handler is not None:
|
||||
self.worker_handler.clear_connector_metadata()
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata) -> None:
|
||||
if self.worker_handler is not None:
|
||||
assert isinstance(kv_connector_metadata, SimpleCPUOffloadMetadata)
|
||||
self.worker_handler.handle_preemptions(kv_connector_metadata)
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
if self.worker_handler is not None:
|
||||
self.worker_handler.start_load_kv()
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
pass # Always load asynchronously, issued in start_load_kv()
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: "AttentionMetadata",
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
pass # Always save asynchronously, issued in wait_for_save()
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
if self.worker_handler is not None:
|
||||
self.worker_handler.wait_for_save()
|
||||
|
||||
def get_finished(
|
||||
self,
|
||||
finished_req_ids: set[str],
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
if self.worker_handler is not None:
|
||||
return self.worker_handler.get_finished(finished_req_ids)
|
||||
return None, None
|
||||
|
||||
def build_connector_worker_meta(self):
|
||||
if self.worker_handler is not None:
|
||||
return self.worker_handler.build_connector_worker_meta()
|
||||
return None
|
||||
|
||||
# --- Scheduler-side methods ---
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
|
||||
if self.scheduler_manager is not None:
|
||||
self.scheduler_manager.bind_gpu_block_pool(gpu_block_pool)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: "Request",
|
||||
blocks: "KVCacheBlocks",
|
||||
num_external_tokens: int,
|
||||
) -> None:
|
||||
if self.scheduler_manager is not None:
|
||||
self.scheduler_manager.update_state_after_alloc(
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.build_connector_meta(scheduler_output)
|
||||
return SimpleCPUOffloadMetadata()
|
||||
|
||||
def update_connector_output(
|
||||
self,
|
||||
connector_output: KVConnectorOutput,
|
||||
) -> None:
|
||||
if self.scheduler_manager is not None:
|
||||
self.scheduler_manager.update_connector_output(connector_output)
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.request_finished(request, block_ids)
|
||||
return False, None
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.request_finished_all_groups(
|
||||
request, block_ids
|
||||
)
|
||||
return False, None
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.has_pending_stores()
|
||||
return False
|
||||
|
||||
def get_boundary_store_stats(self) -> BoundaryStoreStats | None:
|
||||
"""Return cumulative boundary-handoff store diagnostics."""
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.get_boundary_store_stats()
|
||||
return None
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
# Worker-side callers may invoke this hook on connectors constructed
|
||||
# via __new__ (as in test_worker.py), which lack scheduler_manager.
|
||||
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
|
||||
self, "scheduler_manager", None
|
||||
)
|
||||
if scheduler_manager is not None:
|
||||
return scheduler_manager.get_stats()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
SimpleCPUOffloadStats(data=data)
|
||||
if data is not None
|
||||
else SimpleCPUOffloadStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return SimpleCPUOffloadPromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.take_events()
|
||||
return []
|
||||
|
||||
# NOTE: Workers are not contacted. In-flight transfers drain naturally,
|
||||
# and stale completions are ignored by the guarded
|
||||
# SimpleCPUOffloadScheduler._process_store_event().
|
||||
def reset_cache(self) -> bool | None:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.reset()
|
||||
return None
|
||||
@@ -1,235 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mamba conv-state sub-projection decomposition for NIXL transfer.
|
||||
|
||||
With DS conv state layout (dim, state_len), sub-projections are
|
||||
contiguous in memory. Each D rank reads its slices via separate
|
||||
RDMA transfers — no P-side permutation needed.
|
||||
|
||||
Supported model types:
|
||||
- Mamba1: conv = [x], temporal = (intermediate_size, state_size)
|
||||
- Mamba2: conv = [x, B, C], temporal = (num_heads, head_dim)
|
||||
- GDN (Gated Delta Net): conv = [Q, K, V] (dim(Q)==dim(K)),
|
||||
temporal = (num_v_heads, v_dim, k_dim)
|
||||
"""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaConvSplitInfo:
|
||||
"""Per-rank byte sizes of the conv sub-projections.
|
||||
|
||||
Used by both P and D sides for NIXL descriptor registration.
|
||||
All fields are LOCAL to this engine's TP (already divided by TP size).
|
||||
|
||||
DS memory layout within one page (contiguous):
|
||||
Mamba1: |---- x ----| (single sub-projection, no decomposition)
|
||||
Mamba2: |-- x --|- B -|- C -| (B == C)
|
||||
GDN: |- Q -|- K -|-- V --| (dim(Q)==dim(K), V may differ)
|
||||
"""
|
||||
|
||||
conv_rows: int # conv_kernel - 1 (typically 3)
|
||||
# Per-rank column counts per sub-projection:
|
||||
# 1 entry for Mamba1, 3 for Mamba2/GDN.
|
||||
local_proj_dims: tuple[int, ...]
|
||||
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
|
||||
ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes)
|
||||
|
||||
@property
|
||||
def local_conv_dim(self) -> int:
|
||||
"""Total conv columns per rank."""
|
||||
return sum(self.local_proj_dims)
|
||||
|
||||
@property
|
||||
def proj_bytes(self) -> tuple[int, ...]:
|
||||
"""Byte sizes of the sub-projections for one rank."""
|
||||
row_bytes = self.conv_rows * self.conv_dtype_size
|
||||
return tuple(d * row_bytes for d in self.local_proj_dims)
|
||||
|
||||
@property
|
||||
def local_conv_offsets(self) -> list[tuple[int, int]]:
|
||||
"""(byte_offset, byte_size) of each sub-projection within this
|
||||
engine's page.
|
||||
|
||||
Used by both P and D for local descriptor registration.
|
||||
"""
|
||||
offsets: list[tuple[int, int]] = []
|
||||
offset = 0
|
||||
for size in self.proj_bytes:
|
||||
offsets.append((offset, size))
|
||||
offset += size
|
||||
return offsets
|
||||
|
||||
def remote_conv_offsets(
|
||||
self, local_rank_offset: int, tp_ratio: int
|
||||
) -> list[tuple[int, int]]:
|
||||
"""(byte_offset, byte_size) of this D rank's sub-projection slices
|
||||
within one P page.
|
||||
|
||||
Used by D side only, during remote descriptor registration.
|
||||
|
||||
Args:
|
||||
local_rank_offset: which slice this D rank reads.
|
||||
tp_ratio: signed TP ratio.
|
||||
>= 1: D_TP >= P_TP — P page is larger, D reads its slice.
|
||||
< 0: P_TP > D_TP — P pages are smaller, D reads entire
|
||||
P page. Local dims are scaled down by |tp_ratio|
|
||||
to get P-sized offsets.
|
||||
|
||||
"""
|
||||
offsets: list[tuple[int, int]] = []
|
||||
if tp_ratio >= 1:
|
||||
remote_base = 0
|
||||
for size in self.proj_bytes:
|
||||
offsets.append((remote_base + local_rank_offset * size, size))
|
||||
remote_base += size * tp_ratio
|
||||
else:
|
||||
# NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages
|
||||
# are smaller than D's. Local dims are D-sized, but we need
|
||||
# P-sized offsets. Scale down by |tp_ratio|.
|
||||
abs_ratio = -tp_ratio
|
||||
remote_base = 0
|
||||
for size in self.proj_bytes:
|
||||
remote_size = size // abs_ratio
|
||||
offsets.append((remote_base, remote_size))
|
||||
remote_base += remote_size
|
||||
return offsets
|
||||
|
||||
|
||||
def derive_mamba_conv_split(
|
||||
mamba_spec: MambaSpec,
|
||||
local_tp: int,
|
||||
) -> MambaConvSplitInfo:
|
||||
"""Derive per-rank sub-projection byte sizes from a MambaSpec.
|
||||
|
||||
Called once at init on both P and D. Decomposes the conv dimension
|
||||
into its sub-projection parts based on the model type.
|
||||
|
||||
Args:
|
||||
mamba_spec: MambaSpec whose shapes are:
|
||||
shapes[0] = conv state: (conv_dim_local, conv_rows) in DS layout.
|
||||
shapes[1] = temporal state (model-specific shape).
|
||||
local_tp: this engine's tensor-parallel size.
|
||||
|
||||
Returns:
|
||||
MambaConvSplitInfo with per-rank sub-projection dims, conv_rows,
|
||||
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
|
||||
|
||||
"""
|
||||
_supported = (
|
||||
MambaAttentionBackendEnum.MAMBA1,
|
||||
MambaAttentionBackendEnum.MAMBA2,
|
||||
MambaAttentionBackendEnum.GDN_ATTN,
|
||||
)
|
||||
if mamba_spec.mamba_type not in _supported:
|
||||
raise NotImplementedError(
|
||||
f"Conv transfer only supports Mamba1, Mamba2 and GDN models, "
|
||||
f"got mamba_type={mamba_spec.mamba_type!r}."
|
||||
)
|
||||
|
||||
conv_shape = mamba_spec.shapes[0]
|
||||
assert len(conv_shape) == 2, f"Expected 2D conv state shape, got {conv_shape}"
|
||||
|
||||
# NOTE (ZhanqiuHu): 3-read requires DS layout, which is already asserted
|
||||
# in nixl worker __init__. Use it directly instead of heuristic detection.
|
||||
assert is_conv_state_dim_first(), "3-read requires DS conv state layout"
|
||||
local_conv_dim = conv_shape[0] # DS: (conv_dim_local, conv_rows)
|
||||
conv_rows = conv_shape[1]
|
||||
|
||||
conv_dtype_size = torch.tensor(
|
||||
[],
|
||||
dtype=mamba_spec.dtypes[0], # type: ignore[misc]
|
||||
).element_size()
|
||||
ssm_dtype_size = torch.tensor(
|
||||
[],
|
||||
dtype=mamba_spec.dtypes[1], # type: ignore[misc]
|
||||
).element_size()
|
||||
conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size
|
||||
ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size
|
||||
|
||||
local_proj_dims: tuple[int, ...]
|
||||
if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA1:
|
||||
# Mamba1 conv state holds only x (no B/C), so it's a single
|
||||
# contiguous TP shard with no sub-projection decomposition.
|
||||
temporal_shape = mamba_spec.shapes[1]
|
||||
assert temporal_shape[0] == local_conv_dim, (
|
||||
f"Mamba1 temporal state dim ({temporal_shape[0]}) doesn't match "
|
||||
f"conv dim ({local_conv_dim}); both should be "
|
||||
f"intermediate_size/TP."
|
||||
)
|
||||
local_proj_dims = (local_conv_dim,)
|
||||
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2:
|
||||
# NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored
|
||||
# in MambaSpec, so we reconstruct it from the SSM temporal state shape:
|
||||
# shapes[1] = (local_num_heads, head_dim), already divided by TP.
|
||||
head_dim = mamba_spec.shapes[1][1]
|
||||
local_num_heads = mamba_spec.shapes[1][0]
|
||||
intermediate_size = local_num_heads * local_tp * head_dim
|
||||
|
||||
# NOTE (ZhanqiuHu): global conv dim = intermediate_size + 2 * groups_ss,
|
||||
# where groups_ss is the B (= C) dimension. B and C are always the same
|
||||
# size, so we recover groups_ss from the remainder after subtracting x.
|
||||
remainder = local_conv_dim * local_tp - intermediate_size
|
||||
assert remainder > 0 and remainder % 2 == 0, (
|
||||
f"Conv dim ({local_conv_dim}*tp={local_tp}) doesn't decompose "
|
||||
f"into intermediate_size={intermediate_size} + 2*groups_ss. "
|
||||
f"remainder={remainder}"
|
||||
)
|
||||
groups_ss = remainder // 2
|
||||
|
||||
# Divide by TP to get per-rank column counts.
|
||||
x_local = intermediate_size // local_tp
|
||||
b_local = groups_ss // local_tp
|
||||
local_proj_dims = (x_local, b_local, b_local)
|
||||
elif mamba_spec.mamba_type == MambaAttentionBackendEnum.GDN_ATTN:
|
||||
# GDN: conv = [Q, K, V] where dim(Q) == dim(K) == key_dim.
|
||||
# conv_dim = key_dim*2 + value_dim (all global, divided by TP).
|
||||
# Temporal state shape is (num_v_heads/TP, head_v_dim, head_k_dim).
|
||||
temporal_shape = mamba_spec.shapes[1]
|
||||
num_v_heads_local = temporal_shape[0]
|
||||
head_v_dim = temporal_shape[1]
|
||||
value_dim_local = num_v_heads_local * head_v_dim
|
||||
|
||||
remainder = local_conv_dim - value_dim_local
|
||||
assert remainder > 0 and remainder % 2 == 0, (
|
||||
f"GDN conv dim ({local_conv_dim}) doesn't decompose into "
|
||||
f"2*key_dim_local + value_dim_local={value_dim_local}. "
|
||||
f"remainder={remainder}"
|
||||
)
|
||||
key_dim_local = remainder // 2
|
||||
local_proj_dims = (key_dim_local, key_dim_local, value_dim_local)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Conv split not supported for mamba_type={mamba_spec.mamba_type!r}"
|
||||
)
|
||||
|
||||
return MambaConvSplitInfo(
|
||||
conv_rows=conv_rows,
|
||||
local_proj_dims=local_proj_dims,
|
||||
conv_dtype_size=conv_dtype_size,
|
||||
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
|
||||
)
|
||||
|
||||
|
||||
def compute_physical_blocks_per_logical(
|
||||
ssm_sizes: tuple[int, ...], block_len: int
|
||||
) -> int:
|
||||
"""Derive _physical_blocks_per_logical_kv_block from remote metadata.
|
||||
|
||||
The remote engine's ratio is not sent directly in the handshake, so we
|
||||
reconstruct it: total mamba state per logical block / block_len.
|
||||
|
||||
Args:
|
||||
ssm_sizes: (conv_state_bytes, ssm_state_bytes) from NixlAgentMetadata.
|
||||
block_len: the engine's block_len in bytes (from block_lens[0]).
|
||||
|
||||
"""
|
||||
return math.ceil((ssm_sizes[0] + ssm_sizes[1]) / block_len)
|
||||
@@ -1,459 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""IPC-based weight transfer engine using CUDA IPC for communication."""
|
||||
|
||||
import pickle
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Self
|
||||
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
PackedBufferImporter,
|
||||
packed_ipc_consumer,
|
||||
packed_ipc_producer,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class IPCWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side init info for IPC weight transfer. No rendezvous needed.
|
||||
|
||||
`packed` is a must-agree wire param: the trainer ships it here at the init
|
||||
handshake so the worker decodes with the same setting the trainer encoded
|
||||
with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does
|
||||
not need the buffer size (producer-only)."""
|
||||
|
||||
packed: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class IPCTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for IPC weight transfer. No rendezvous needed;
|
||||
`rank` (from `TrainerInitInfo`) identifies this trainer process — rank 0
|
||||
ships the merged IPC handles. All ranks still join the handle all-gather.
|
||||
|
||||
`packed` / `packed_buffer_size_bytes` are the transfer's wire params. The
|
||||
trainer propagates `packed` to the worker at `trainer_init` so the two sides
|
||||
cannot disagree. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "ipc"
|
||||
packed: bool = False
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
|
||||
|
||||
@dataclass
|
||||
class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Per-round update info for the IPC weight transfer backend."""
|
||||
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
ipc_handles: list[dict[str, tuple]] | dict[str, tuple] | None = None
|
||||
"""IPC handles mapping physical GPU UUID to rebuild_cuda_tensor args.
|
||||
For non-packed mode: list of per-parameter handle dicts.
|
||||
For packed mode: single handle dict for the packed buffer."""
|
||||
ipc_handles_pickled: str | None = None
|
||||
"""Base64-encoded pickled IPC handles, used for HTTP transport."""
|
||||
tensor_sizes: list[int] | None = None
|
||||
"""Per-parameter sizes in bytes within the packed buffer.
|
||||
Required when packed=True, unused otherwise."""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.ipc_handles_pickled is not None:
|
||||
if self.ipc_handles is not None:
|
||||
raise ValueError(
|
||||
"Cannot specify both `ipc_handles` and `ipc_handles_pickled`"
|
||||
)
|
||||
|
||||
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
|
||||
raise ValueError(
|
||||
"Refusing to deserialize `ipc_handles_pickled` without "
|
||||
"VLLM_ALLOW_INSECURE_SERIALIZATION=1"
|
||||
)
|
||||
|
||||
self.ipc_handles = pickle.loads(base64.b64decode(self.ipc_handles_pickled))
|
||||
self.ipc_handles_pickled = None
|
||||
|
||||
if self.ipc_handles is None:
|
||||
raise ValueError(
|
||||
"Either `ipc_handles` or `ipc_handles_pickled` must be provided"
|
||||
)
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
f"`dtype_names` should be of the same size as `names`: "
|
||||
f"got {len(self.dtype_names)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.shapes) != num_params:
|
||||
raise ValueError(
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
# Unpacked transfers carry a per-parameter handle dict (a list); packed
|
||||
# transfers carry a single dict for the whole buffer, so the list check
|
||||
# only applies to the list case.
|
||||
if isinstance(self.ipc_handles, list) and len(self.ipc_handles) != num_params:
|
||||
raise ValueError(
|
||||
f"`ipc_handles` should be of the same size as `names`: "
|
||||
f"got {len(self.ipc_handles)} and {len(self.names)}"
|
||||
)
|
||||
|
||||
|
||||
class IPCWeightTransferEngine(
|
||||
WeightTransferEngine[IPCWeightTransferInitInfo, IPCWeightTransferUpdateInfo]
|
||||
):
|
||||
"""Weight transfer engine using CUDA IPC between trainer and workers.
|
||||
|
||||
This implementation uses CUDA IPC to transfer weights from the trainer (rank 0)
|
||||
to all inference workers in a process group. IPC handles are used to share
|
||||
memory between processes on the same node.
|
||||
"""
|
||||
|
||||
# Define backend-specific dataclass types
|
||||
init_info_cls = IPCWeightTransferInitInfo
|
||||
update_info_cls = IPCWeightTransferUpdateInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
# Set from the trainer-supplied init info at the handshake; defaults are
|
||||
# only for the (unreachable) receive-before-init case.
|
||||
self.packed = False
|
||||
# Shared across all chunks of every packed transfer this engine
|
||||
# receives; see PackedBufferImporter for the refcount contract.
|
||||
self._packed_importer = PackedBufferImporter()
|
||||
|
||||
def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None:
|
||||
"""Initialize the weight transfer mechanism. No data-plane rendezvous is
|
||||
needed for IPC; this just records the trainer-supplied wire params so the
|
||||
worker decodes exactly as the trainer encoded.
|
||||
|
||||
Args:
|
||||
init_info: IPC initialization info (carries `packed`).
|
||||
|
||||
"""
|
||||
self.packed = init_info.packed
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Initialize layerwise reloading for the incoming checkpoint weights."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading after all weights have been received."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
# Every reduce_tensor call is a fresh export with its own refcount
|
||||
# slot, so releasing once per update always balances this update's
|
||||
# export and lets the trainer reclaim its staging buffer. Callers
|
||||
# that skip finish are still covered by the replace-on-next-export
|
||||
# path inside the importer.
|
||||
self._packed_importer.close()
|
||||
|
||||
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
|
||||
"""Receive weights from the trainer via CUDA IPC handles and load them.
|
||||
|
||||
Whether the transfer is packed is read from `self.packed`, set at the
|
||||
init handshake from the trainer's init info, so it is guaranteed to
|
||||
match how the trainer encoded.
|
||||
|
||||
Args:
|
||||
update_info: IPC update info containing parameter names, dtypes, shapes,
|
||||
and IPC handles. Each IPC handle is a mapping between physical
|
||||
GPU UUID and the rebuild_cuda_tensor args tuple.
|
||||
|
||||
"""
|
||||
# Use the worker's assigned device rather than the ambient current
|
||||
# device: the receive path is no longer wrapped in
|
||||
# `with torch.device(self.device)` by the caller, so the current device
|
||||
# is not guaranteed to match self.device. The IPC tensors must be
|
||||
# rebuilt on the device the model lives on.
|
||||
device_index = self.device.index
|
||||
|
||||
if self.packed:
|
||||
if update_info.tensor_sizes is None:
|
||||
raise ValueError("`tensor_sizes` is required when packed=True")
|
||||
assert isinstance(update_info.ipc_handles, dict)
|
||||
weights = packed_ipc_consumer(
|
||||
ipc_handle=update_info.ipc_handles,
|
||||
names=update_info.names,
|
||||
shapes=update_info.shapes,
|
||||
dtype_names=update_info.dtype_names,
|
||||
tensor_sizes=update_info.tensor_sizes,
|
||||
device_index=device_index,
|
||||
importer=self._packed_importer,
|
||||
)
|
||||
else:
|
||||
assert isinstance(update_info.ipc_handles, list)
|
||||
weights = []
|
||||
for name, ipc_handle in zip(
|
||||
update_info.names,
|
||||
update_info.ipc_handles,
|
||||
):
|
||||
props = torch.cuda.get_device_properties(device_index)
|
||||
physical_gpu_id = str(props.uuid)
|
||||
|
||||
if physical_gpu_id not in ipc_handle:
|
||||
raise ValueError(
|
||||
f"IPC handle not found for GPU UUID "
|
||||
f"{physical_gpu_id}. "
|
||||
f"Available UUIDs: {list(ipc_handle.keys())}"
|
||||
)
|
||||
|
||||
args = ipc_handle[physical_gpu_id]
|
||||
list_args = list(args)
|
||||
# Index 6 of the args from reduce_tensor is the device_index.
|
||||
# We need to overwrite it with the receiver's device index.
|
||||
list_args[6] = device_index
|
||||
weight = rebuild_cuda_tensor(*list_args)
|
||||
weights.append((name, weight))
|
||||
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
disable_mtp_completeness_check,
|
||||
)
|
||||
|
||||
with disable_mtp_completeness_check():
|
||||
self.model.load_weights(weights)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._packed_importer.close()
|
||||
|
||||
|
||||
class IPCTrainerWeightTransferEngine(TrainerWeightTransferEngine[IPCTrainerInitInfo]):
|
||||
"""Trainer-side CUDA IPC weight transfer engine.
|
||||
|
||||
Called on every trainer rank. For multi-rank (e.g. FSDP) trainers all ranks
|
||||
iterate the source (materializing each tensor) and contribute to the
|
||||
IPC-handle all-gather; only the sender (rank 0) ships the merged handles to
|
||||
the inference side. IPC transfer is straight-line (no concurrent broadcast
|
||||
like NCCL): `update_weights` *is* the transfer, and it rides the client, so
|
||||
it no-ops on non-senders.
|
||||
|
||||
`packed` / `packed_buffer_size_bytes` come from `IPCTrainerInitInfo`; the
|
||||
sender propagates `packed` to the worker at `trainer_init`.
|
||||
"""
|
||||
|
||||
init_info_cls = IPCTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource,
|
||||
is_sender: bool = True,
|
||||
packed: bool = False,
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
) -> None:
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.packed = packed
|
||||
self.packed_buffer_size_bytes = packed_buffer_size_bytes
|
||||
self.device_index = torch.accelerator.current_device_index()
|
||||
self.gpu_uuid = str(torch.cuda.get_device_properties(self.device_index).uuid)
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: IPCTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
if source is None:
|
||||
raise ValueError("IPC trainer weight transfer requires a WeightSource.")
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=source,
|
||||
is_sender=init_info.is_sender,
|
||||
packed=init_info.packed,
|
||||
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
|
||||
)
|
||||
# IPC needs no data-plane rendezvous. The sender ships the must-agree
|
||||
# `packed` flag so the worker decodes exactly as this trainer encodes.
|
||||
if engine.is_sender:
|
||||
engine.client.init_weight_transfer_engine({"packed": init_info.packed})
|
||||
return engine
|
||||
|
||||
def send_weights(self) -> None:
|
||||
assert self.source is not None
|
||||
source = self.source
|
||||
if self.is_sender:
|
||||
self.client.start_weight_update()
|
||||
# Unpacked returns strong refs to its IPC-shared copies; they must stay
|
||||
# alive across `finish_weight_update` too.
|
||||
weight_refs = self._send(source)
|
||||
if self.is_sender:
|
||||
self.client.finish_weight_update()
|
||||
self._post_send_sync()
|
||||
del weight_refs
|
||||
|
||||
# ---- data plane (runs on all ranks; only the sender ships) ----
|
||||
|
||||
def _send(self, source: WeightSource) -> list[torch.Tensor] | None:
|
||||
if self.packed:
|
||||
self._send_packed(source)
|
||||
return None
|
||||
return self._send_unpacked(source)
|
||||
|
||||
def _all_gather_and_merge_handles(
|
||||
self,
|
||||
handles: list[dict[str, tuple]],
|
||||
) -> list[dict[str, tuple]]:
|
||||
"""All-gather and merge IPC handle dicts across ranks in one call.
|
||||
|
||||
Each rank contributes a list of {gpu_uuid: ipc_args} dicts (one
|
||||
per parameter or one per chunk). A single all_gather_object
|
||||
collects every rank's full list, then the sender merges per-index so
|
||||
each dict maps every GPU UUID to its args.
|
||||
|
||||
The all-gather runs over the *default* process group; this assumes the
|
||||
default group is exactly the set of colocated trainer ranks and that the
|
||||
sender is a member. No-op (returns handles unchanged) when no distributed
|
||||
group exists.
|
||||
"""
|
||||
if (
|
||||
not torch.distributed.is_initialized()
|
||||
or torch.distributed.get_world_size() == 1
|
||||
):
|
||||
return handles
|
||||
|
||||
world_size = torch.distributed.get_world_size()
|
||||
gathered: list[list[dict[str, tuple]] | None] = [None] * world_size
|
||||
torch.distributed.all_gather_object(gathered, handles)
|
||||
torch.distributed.barrier()
|
||||
torch.cuda.synchronize()
|
||||
|
||||
if self.is_sender:
|
||||
merged: list[dict[str, tuple]] = []
|
||||
for param_idx in range(len(handles)):
|
||||
m: dict[str, tuple] = {}
|
||||
for rank_handles in gathered:
|
||||
if rank_handles is not None:
|
||||
m.update(rank_handles[param_idx])
|
||||
merged.append(m)
|
||||
return merged
|
||||
return [{} for _ in handles]
|
||||
|
||||
@staticmethod
|
||||
def _post_send_sync() -> None:
|
||||
"""Barrier + ipc_collect after a send; no-op if single-GPU."""
|
||||
if (
|
||||
torch.distributed.is_initialized()
|
||||
and torch.distributed.get_world_size() > 1
|
||||
):
|
||||
torch.distributed.barrier()
|
||||
torch.cuda.ipc_collect()
|
||||
|
||||
def _send_unpacked(self, source: WeightSource) -> list[torch.Tensor]:
|
||||
"""Iterate the source, build one IPC handle per param, all-gather the
|
||||
handles across ranks, and (sender) ship them in one update call.
|
||||
|
||||
Returns the strong refs to every contiguous copy. reduce_tensor's args
|
||||
do NOT keep storage alive, and non-contiguous inputs allocate fresh
|
||||
storage in .contiguous(); the caller must keep these alive until the
|
||||
post-send barrier (past `finish`) so the consumer's IPC views stay valid.
|
||||
"""
|
||||
names: list[str] = []
|
||||
dtype_names: list[str] = []
|
||||
shapes: list[list[int]] = []
|
||||
ipc_handles: list[dict[str, tuple]] = []
|
||||
weight_refs: list[torch.Tensor] = []
|
||||
|
||||
for name, tensor in source:
|
||||
# FSDP shards were gathered by the source; ensure contiguity here.
|
||||
names.append(name)
|
||||
dtype_names.append(str(tensor.dtype).split(".")[-1])
|
||||
shapes.append(list(tensor.shape))
|
||||
|
||||
weight = tensor.detach().contiguous()
|
||||
weight_refs.append(weight)
|
||||
_, ipc_args = reduce_tensor(weight)
|
||||
ipc_handles.append({self.gpu_uuid: ipc_args})
|
||||
|
||||
ipc_handles = self._all_gather_and_merge_handles(ipc_handles)
|
||||
self._do_send(
|
||||
names=names,
|
||||
dtype_names=dtype_names,
|
||||
shapes=shapes,
|
||||
ipc_handles=ipc_handles,
|
||||
)
|
||||
return weight_refs
|
||||
|
||||
def _send_packed(self, source: WeightSource) -> None:
|
||||
"""Send weights in bounded-memory chunks (packed mode)."""
|
||||
for chunk in packed_ipc_producer(
|
||||
iterator=iter(source),
|
||||
gpu_uuid=self.gpu_uuid,
|
||||
post_iter_func=lambda item: item[1],
|
||||
buffer_size_bytes=self.packed_buffer_size_bytes,
|
||||
):
|
||||
ipc_handle = self._all_gather_and_merge_handles([chunk.ipc_handle])[0]
|
||||
self._do_send(
|
||||
names=chunk.names,
|
||||
dtype_names=chunk.dtype_names,
|
||||
shapes=chunk.shapes,
|
||||
ipc_handles=ipc_handle,
|
||||
tensor_sizes=chunk.tensor_sizes,
|
||||
)
|
||||
# Per-chunk barrier: the producer reuses a single IPC buffer across
|
||||
# chunks, but only the sender waits for the consumers (via _do_send).
|
||||
# Without syncing every rank here, non-sender ranks race ahead and
|
||||
# overwrite their buffer while their colocated worker is still
|
||||
# reading the current chunk, silently corrupting the transfer.
|
||||
self._post_send_sync()
|
||||
|
||||
def _do_send(
|
||||
self,
|
||||
names: list[str],
|
||||
dtype_names: list[str],
|
||||
shapes: list[list[int]],
|
||||
ipc_handles: list[dict[str, tuple]] | dict[str, tuple],
|
||||
tensor_sizes: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Build one update payload and ship it via the client. Only the sender
|
||||
ships (non-sender ranks already contributed to the handle all-gather).
|
||||
|
||||
Emits raw `ipc_handles`; transports that cannot carry them natively
|
||||
(HTTP/JSON) pickle them in their client (see `HTTPVLLMWeightSyncClient`).
|
||||
"""
|
||||
if not self.is_sender:
|
||||
return
|
||||
update_fields: dict[str, Any] = {
|
||||
"names": names,
|
||||
"dtype_names": dtype_names,
|
||||
"shapes": shapes,
|
||||
"ipc_handles": ipc_handles,
|
||||
}
|
||||
if tensor_sizes is not None:
|
||||
update_fields["tensor_sizes"] = tensor_sizes
|
||||
|
||||
update_info = IPCWeightTransferUpdateInfo(**update_fields)
|
||||
self.client.update_weights(asdict(update_info))
|
||||
@@ -1,864 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for the NCCL M2N (`nccl_m2n`) weight transfer backend.
|
||||
|
||||
M2N reshards a tensor between two disjoint meshes of ranks that live in one
|
||||
communicator: the trainer occupies ranks `[0, T)` and the inference workers
|
||||
`[T, T + N)`. This module holds everything both sides need — the optional
|
||||
runtime import, layout descriptors, and the conversion to `nccl.m2n` types —
|
||||
so the engine module stays about the transfer itself.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import ParamMeta
|
||||
from vllm.utils.nccl import unpinned_nccl_env
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
# Placement code for a mesh axis that replicates. Any other (non-negative) code
|
||||
# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so
|
||||
# layouts survive the JSON init handshake without a custom encoder.
|
||||
REPLICATE = -1
|
||||
|
||||
# `NCCL_RESHARD_MAX_TENSOR_DIMS`; 4-D and higher are rejected by the library.
|
||||
MAX_TENSOR_DIMS = 3
|
||||
|
||||
MESH_NDIMS = 2
|
||||
|
||||
# Destination mesh convention. Keeping these roles named avoids scattering
|
||||
# positional axis assumptions through the worker-side planner.
|
||||
DESTINATION_REPLICA_AXIS = 0
|
||||
DESTINATION_SHARD_AXIS = 1
|
||||
|
||||
# Conservative capacities accepted by the supported nccl_m2n algorithms. They
|
||||
# are not exposed by its Python API yet; remove these mirrors once nccl_m2n
|
||||
# provides communication-free plan validation.
|
||||
MAX_SOURCE_SHARDS = 16
|
||||
MAX_DEST_SHARDS = 64
|
||||
|
||||
# Increment when the JSON init payload changes incompatibly.
|
||||
M2N_WIRE_SCHEMA_VERSION = 2
|
||||
|
||||
SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset(
|
||||
dtype
|
||||
for dtype in (
|
||||
getattr(torch, name, None)
|
||||
for name in (
|
||||
"int8",
|
||||
"uint8",
|
||||
"float8_e4m3fn",
|
||||
"float8_e5m2",
|
||||
"float16",
|
||||
"bfloat16",
|
||||
"int32",
|
||||
"uint32",
|
||||
"float32",
|
||||
"int64",
|
||||
"uint64",
|
||||
"float64",
|
||||
)
|
||||
)
|
||||
if dtype is not None
|
||||
)
|
||||
|
||||
_IMPORT_HINT = (
|
||||
"The nccl_m2n weight transfer backend requires the `nccl-extensions` "
|
||||
"package (and its `nccl4py` dependency), which is not installed by vLLM. "
|
||||
"See https://github.com/NVIDIA/nccl-extensions for build "
|
||||
"and install instructions. It needs NCCL 2.30.5 or newer."
|
||||
)
|
||||
|
||||
|
||||
def prepare_m2n_local_runtime(m2n: Any, max_cta: int | None) -> tuple[Any, int, Any]:
|
||||
"""Prepare rank-local resources before any rank enters NCCL init."""
|
||||
import nccl.core as nccl
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
if not isinstance(device, int) or isinstance(device, bool) or device < 0:
|
||||
raise RuntimeError(f"invalid current accelerator device index: {device!r}")
|
||||
handle = m2n.Handle.create(m2n.Config(max_cta=max_cta))
|
||||
return nccl, device, handle
|
||||
|
||||
|
||||
def create_m2n_communicator(
|
||||
nccl: Any,
|
||||
group: "StatelessProcessGroup",
|
||||
unique_id_bytes: bytes | None,
|
||||
) -> Any:
|
||||
"""Create the M2N data-plane communicator through NCCL4Py."""
|
||||
if unique_id_bytes is None:
|
||||
envelope: dict[str, bytes | str | None] | None = None
|
||||
if group.rank == 0:
|
||||
try:
|
||||
envelope = {
|
||||
"unique_id": bytes(nccl.get_unique_id().as_bytes),
|
||||
"error": None,
|
||||
}
|
||||
except Exception as exc:
|
||||
envelope = {
|
||||
"unique_id": None,
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
}
|
||||
envelope = group.broadcast_obj(envelope, src=0)
|
||||
if not isinstance(envelope, Mapping) or set(envelope) != {"unique_id", "error"}:
|
||||
raise RuntimeError("invalid NCCL4Py unique-id envelope")
|
||||
error = envelope["error"]
|
||||
received_unique_id = envelope["unique_id"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
raise RuntimeError("invalid NCCL4Py unique-id error")
|
||||
raise RuntimeError(f"NCCL4Py unique-id generation failed: {error}")
|
||||
if not isinstance(received_unique_id, bytes):
|
||||
raise RuntimeError("NCCL4Py unique-id broadcast did not return bytes")
|
||||
unique_id_bytes = received_unique_id
|
||||
|
||||
unique_id = nccl.UniqueId.from_bytes(unique_id_bytes)
|
||||
with unpinned_nccl_env():
|
||||
return nccl.Communicator.init(
|
||||
nranks=group.world_size,
|
||||
rank=group.rank,
|
||||
unique_id=unique_id,
|
||||
)
|
||||
|
||||
|
||||
def import_m2n() -> Any:
|
||||
"""Import `nccl.m2n` lazily, with an actionable error when it is missing.
|
||||
|
||||
Deferred so that importing vLLM — or any other weight transfer backend —
|
||||
never requires the m2n runtime to be present.
|
||||
"""
|
||||
try:
|
||||
import nccl.m2n as m2n
|
||||
except ImportError as e:
|
||||
raise ImportError(f"{_IMPORT_HINT} (import failed: {e})") from e
|
||||
return m2n
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NMesh:
|
||||
"""One side's rank topology — pure topology, no tensor placement.
|
||||
|
||||
Mirrors `ncclMesh_t`: a 2-axis mesh owning the contiguous rank interval
|
||||
`[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a
|
||||
single-axis topology is spelled with a second axis of size 1.
|
||||
|
||||
A model may use a different factorization for each tensor. For example,
|
||||
dense weights can use a DP x TP mesh while expert weights use EDP x EP.
|
||||
"""
|
||||
|
||||
dims: tuple[int, int]
|
||||
start_rank: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "dims", tuple(self.dims))
|
||||
if len(self.dims) != MESH_NDIMS or any(
|
||||
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
|
||||
for dim in self.dims
|
||||
):
|
||||
raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints")
|
||||
if (
|
||||
not isinstance(self.start_rank, int)
|
||||
or isinstance(self.start_rank, bool)
|
||||
or self.start_rank < 0
|
||||
):
|
||||
raise ValueError(
|
||||
f"mesh start_rank must be non-negative, got {self.start_rank}"
|
||||
)
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self.dims[0] * self.dims[1]
|
||||
|
||||
|
||||
# A tensor's placement over its side's mesh: one code per mesh axis
|
||||
# (`REPLICATE`, or the tensor dim that axis shards), or `REPLICATED` for a
|
||||
# tensor every rank holds in full.
|
||||
Placements = tuple[int, int]
|
||||
REPLICATED: Placements | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NLayout:
|
||||
"""A tensor's mesh and placement on one side of a transfer."""
|
||||
|
||||
mesh: M2NMesh
|
||||
placements: Placements | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.mesh, M2NMesh):
|
||||
raise TypeError("M2N layout mesh must be an M2NMesh")
|
||||
if self.placements is not None:
|
||||
object.__setattr__(self, "placements", tuple(self.placements))
|
||||
|
||||
|
||||
def check_placements(placements: Placements, context: str = "placements") -> None:
|
||||
"""Reject placement pairs nccl_m2n cannot express."""
|
||||
if len(placements) != MESH_NDIMS:
|
||||
raise ValueError(f"{context} must have {MESH_NDIMS} entries")
|
||||
if any(not isinstance(code, int) or isinstance(code, bool) for code in placements):
|
||||
raise TypeError(f"{context} must contain integer placement codes")
|
||||
invalid = [code for code in placements if code < REPLICATE]
|
||||
if invalid:
|
||||
raise ValueError(
|
||||
f"{context} contains invalid placement code {invalid[0]}; valid "
|
||||
f"codes are {REPLICATE} (Replicate) and non-negative tensor dimensions"
|
||||
)
|
||||
num_sharded = sum(code != REPLICATE for code in placements)
|
||||
if num_sharded == 0:
|
||||
raise ValueError(
|
||||
f"{context}: a fully replicated tensor is carried as REPLICATED, "
|
||||
"not as two REPLICATE axes"
|
||||
)
|
||||
if num_sharded == MESH_NDIMS:
|
||||
raise ValueError(
|
||||
f"{context}: nccl_m2n needs one REPLICATE mesh axis, but "
|
||||
f"{placements} shards both"
|
||||
)
|
||||
|
||||
|
||||
def resolve_layout(
|
||||
mesh: M2NMesh,
|
||||
placements: Placements | None,
|
||||
context: str = "placements",
|
||||
) -> tuple[M2NMesh, Placements]:
|
||||
"""Pair one tensor's placement with the mesh m2n should see for it.
|
||||
|
||||
A replicated tensor needs a size-1 mesh axis to carry a no-op shard, since
|
||||
m2n has no `{REPLICATE, REPLICATE}`. It is therefore described over the
|
||||
*same rank interval* re-factored as `(size, 1)`. That re-factoring is sound
|
||||
precisely because replication is order-independent — every rank holds the
|
||||
whole tensor, so it does not matter that `(a, b)` and `(size, 1)` walk the
|
||||
interval in a different order. A sharded tensor keeps its side's own
|
||||
factorization, where rank order decides who owns which shard.
|
||||
"""
|
||||
if placements is None:
|
||||
return M2NMesh((mesh.size, 1), mesh.start_rank), (REPLICATE, 0)
|
||||
check_placements(placements, context)
|
||||
return mesh, placements
|
||||
|
||||
|
||||
def validate_layout(
|
||||
mesh: M2NMesh,
|
||||
placements: Placements,
|
||||
shape: Sequence[int],
|
||||
context: str,
|
||||
) -> tuple[int, ...]:
|
||||
"""Validate one resolved layout and return its required local shape."""
|
||||
local_shape = list(shape)
|
||||
for axis, tensor_dim in enumerate(placements):
|
||||
if tensor_dim == REPLICATE:
|
||||
continue
|
||||
if tensor_dim >= len(shape):
|
||||
raise ValueError(
|
||||
f"{context} shards tensor dim {tensor_dim}, but parameter has "
|
||||
f"rank {len(shape)}"
|
||||
)
|
||||
factor = mesh.dims[axis]
|
||||
if shape[tensor_dim] % factor:
|
||||
raise ValueError(
|
||||
f"{context} shards dim {tensor_dim} (size {shape[tensor_dim]}) "
|
||||
f"over {factor} ranks, which does not divide evenly"
|
||||
)
|
||||
local_shape[tensor_dim] //= factor
|
||||
return tuple(local_shape)
|
||||
|
||||
|
||||
def expected_local_shape(meta: "M2NParamMeta") -> tuple[int, ...]:
|
||||
"""Validate a parameter's source layout and return its local shape."""
|
||||
mesh, placements = resolve_layout(
|
||||
meta.source_layout.mesh,
|
||||
meta.source_layout.placements,
|
||||
f"parameter '{meta.name}' source placements",
|
||||
)
|
||||
return validate_layout(
|
||||
mesh, placements, meta.shape, f"parameter '{meta.name}' source layout"
|
||||
)
|
||||
|
||||
|
||||
def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None:
|
||||
"""Reject parameters nccl_m2n cannot transfer."""
|
||||
if dtype not in SUPPORTED_DTYPES:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not "
|
||||
f"support. Supported: {sorted(str(d) for d in SUPPORTED_DTYPES)}"
|
||||
)
|
||||
if not 1 <= len(shape) <= MAX_TENSOR_DIMS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' has rank {len(shape)}; nccl_m2n supports "
|
||||
f"rank 1..{MAX_TENSOR_DIMS}"
|
||||
)
|
||||
|
||||
|
||||
def _shard_spec(mesh: M2NMesh, placements: Placements) -> tuple[int, int]:
|
||||
return next(
|
||||
(code, mesh.dims[axis])
|
||||
for axis, code in enumerate(placements)
|
||||
if code != REPLICATE
|
||||
)
|
||||
|
||||
|
||||
def _max_partition_overlaps(
|
||||
extent: int,
|
||||
partitions: int,
|
||||
other_partitions: int,
|
||||
) -> int:
|
||||
"""Return the maximum other partitions intersected by any one partition."""
|
||||
if extent == 0:
|
||||
return 0
|
||||
width = extent // partitions
|
||||
other_width = extent // other_partitions
|
||||
max_overlaps = 0
|
||||
for index in range(partitions):
|
||||
start = index * width
|
||||
end = start + width
|
||||
first_other = start // other_width
|
||||
last_other = (end - 1) // other_width
|
||||
max_overlaps = max(max_overlaps, last_other - first_other + 1)
|
||||
return max_overlaps
|
||||
|
||||
|
||||
def check_plan_limits(
|
||||
src: tuple[M2NMesh, Placements],
|
||||
dst: tuple[M2NMesh, Placements],
|
||||
name: str,
|
||||
shape: Sequence[int],
|
||||
) -> None:
|
||||
"""Reject plans whose per-shard fan-in or fan-out exceeds M2N capacity."""
|
||||
src_dim, src_shards = _shard_spec(*src)
|
||||
dst_dim, dst_shards = _shard_spec(*dst)
|
||||
if any(extent == 0 for extent in shape):
|
||||
max_sources = max_targets = 0
|
||||
elif src_dim != dst_dim:
|
||||
max_sources = src_shards
|
||||
max_targets = dst_shards
|
||||
else:
|
||||
extent = shape[src_dim]
|
||||
max_sources = _max_partition_overlaps(extent, dst_shards, src_shards)
|
||||
max_targets = _max_partition_overlaps(extent, src_shards, dst_shards)
|
||||
|
||||
if max_sources > MAX_SOURCE_SHARDS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' would feed up to {max_sources} source shards "
|
||||
f"into one destination shard, over nccl_m2n MAX_SOURCES="
|
||||
f"{MAX_SOURCE_SHARDS}"
|
||||
)
|
||||
if max_targets > MAX_DEST_SHARDS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' would fan one source shard out to up to "
|
||||
f"{max_targets} destination shards, over nccl_m2n "
|
||||
f"MAX_TARGETS={MAX_DEST_SHARDS}"
|
||||
)
|
||||
|
||||
|
||||
def validate_transfer_plan(
|
||||
metas: Sequence["M2NParamMeta"],
|
||||
dst_mesh: M2NMesh,
|
||||
dst_placements: Sequence[Placements | None],
|
||||
) -> None:
|
||||
"""Validate the complete ordered source-to-destination transfer plan."""
|
||||
if len(dst_placements) != len(metas):
|
||||
raise ValueError(
|
||||
f"destination plan has {len(dst_placements)} parameters, expected "
|
||||
f"{len(metas)}"
|
||||
)
|
||||
for meta, placement in zip(metas, dst_placements):
|
||||
check_transferable(meta.name, meta.dtype, meta.shape)
|
||||
src = resolve_layout(
|
||||
meta.source_layout.mesh,
|
||||
meta.source_layout.placements,
|
||||
f"parameter '{meta.name}' source placements",
|
||||
)
|
||||
validate_layout(*src, meta.shape, f"parameter '{meta.name}' source layout")
|
||||
dst = resolve_layout(
|
||||
dst_mesh,
|
||||
placement,
|
||||
f"parameter '{meta.name}' destination placements",
|
||||
)
|
||||
validate_layout(*dst, meta.shape, f"parameter '{meta.name}' destination layout")
|
||||
check_plan_limits(src, dst, meta.name, meta.shape)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NParamMeta(ParamMeta):
|
||||
"""`ParamMeta` extended with how the trainer places this tensor.
|
||||
|
||||
The base class carries only name / dtype / full shape, which is not enough
|
||||
to plan a reshard. `source_layout` carries both the per-parameter mesh and
|
||||
its placement on that mesh.
|
||||
"""
|
||||
|
||||
source_layout: M2NLayout
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.source_layout, M2NLayout):
|
||||
raise TypeError("source_layout must be an M2NLayout")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NWireParam:
|
||||
"""JSON-safe source metadata for one parameter in the init handshake."""
|
||||
|
||||
name: str
|
||||
dtype_name: str
|
||||
shape: tuple[int, ...]
|
||||
src_mesh_dims: tuple[int, int]
|
||||
src_placements: Placements | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
shape = tuple(self.shape)
|
||||
dims = tuple(self.src_mesh_dims)
|
||||
placements = None if self.src_placements is None else tuple(self.src_placements)
|
||||
object.__setattr__(self, "shape", shape)
|
||||
object.__setattr__(self, "src_mesh_dims", dims)
|
||||
object.__setattr__(self, "src_placements", placements)
|
||||
if not isinstance(self.name, str) or not self.name:
|
||||
raise ValueError("M2N wire parameter name must not be empty")
|
||||
if not isinstance(self.dtype_name, str) or not self.dtype_name:
|
||||
raise ValueError(f"parameter '{self.name}' has an empty wire dtype name")
|
||||
if not shape or any(
|
||||
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
|
||||
for dim in shape
|
||||
):
|
||||
raise ValueError(
|
||||
f"parameter '{self.name}' shape must contain positive integers"
|
||||
)
|
||||
M2NMesh(cast(tuple[int, int], dims), 0)
|
||||
if placements is not None:
|
||||
check_placements(
|
||||
cast(Placements, placements),
|
||||
f"parameter '{self.name}' source placements",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: Mapping[str, Any]) -> "M2NWireParam":
|
||||
"""Parse one strict wire record from its JSON-decoded mapping."""
|
||||
fields = {
|
||||
"name",
|
||||
"dtype_name",
|
||||
"shape",
|
||||
"src_mesh_dims",
|
||||
"src_placements",
|
||||
}
|
||||
missing = fields - value.keys()
|
||||
extra = value.keys() - fields
|
||||
if missing or extra:
|
||||
details = []
|
||||
if missing:
|
||||
details.append(f"missing {sorted(missing)}")
|
||||
if extra:
|
||||
details.append(f"unexpected {sorted(extra)}")
|
||||
raise ValueError(f"invalid M2N wire parameter: {', '.join(details)}")
|
||||
return cls(**value)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Serialize this record using only JSON-compatible values."""
|
||||
return {
|
||||
"name": self.name,
|
||||
"dtype_name": self.dtype_name,
|
||||
"shape": list(self.shape),
|
||||
"src_mesh_dims": list(self.src_mesh_dims),
|
||||
"src_placements": (
|
||||
None if self.src_placements is None else list(self.src_placements)
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def source_plan_digest(params: Sequence[M2NWireParam]) -> str:
|
||||
"""Return a rank-independent digest of the ordered source wire plan."""
|
||||
payload = json.dumps(
|
||||
{
|
||||
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
||||
"params": [param.to_dict() for param in params],
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def complete_plan_digest(
|
||||
source_digest: str,
|
||||
dst_mesh: M2NMesh,
|
||||
dst_placements: Sequence[Placements | None],
|
||||
) -> str:
|
||||
"""Return a digest of the agreed source and destination plan."""
|
||||
if not isinstance(source_digest, str) or not source_digest:
|
||||
raise ValueError("source digest must be a non-empty string")
|
||||
payload = json.dumps(
|
||||
{
|
||||
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
||||
"source_digest": source_digest,
|
||||
"destination_mesh": {
|
||||
"dims": list(dst_mesh.dims),
|
||||
"start_rank": dst_mesh.start_rank,
|
||||
},
|
||||
"destination_placements": [
|
||||
None if placement is None else list(placement)
|
||||
for placement in dst_placements
|
||||
],
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def check_source_plan_agreement(
|
||||
group: "StatelessProcessGroup",
|
||||
params: Sequence[M2NWireParam],
|
||||
expected_digest: str | None = None,
|
||||
local_error: str | None = None,
|
||||
) -> str:
|
||||
"""Verify every participant entered with the same source plan."""
|
||||
prepared_error = local_error
|
||||
digest: str | None = None
|
||||
declared: str | None = expected_digest
|
||||
if prepared_error is None:
|
||||
try:
|
||||
digest = source_plan_digest(params)
|
||||
declared = digest if expected_digest is None else expected_digest
|
||||
if not isinstance(declared, str) or not declared:
|
||||
raise ValueError("declared source digest must be a non-empty string")
|
||||
except Exception as exc:
|
||||
prepared_error = f"{type(exc).__name__}: {exc}"
|
||||
elif not isinstance(prepared_error, str) or not prepared_error:
|
||||
prepared_error = "TypeError: local source error must be a non-empty string"
|
||||
|
||||
gathered = group.all_gather_obj(
|
||||
{
|
||||
"phase": "source",
|
||||
"digest": digest,
|
||||
"declared_digest": declared,
|
||||
"error": prepared_error,
|
||||
}
|
||||
)
|
||||
errors: list[str] = []
|
||||
expected_fields = {"phase", "digest", "declared_digest", "error"}
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
agreed: list[tuple[int, str, str]] = []
|
||||
for rank, item in enumerate(gathered):
|
||||
if not isinstance(item, Mapping) or set(item) != expected_fields:
|
||||
errors.append(f"rank {rank}: invalid source preflight envelope")
|
||||
continue
|
||||
if item["phase"] != "source":
|
||||
errors.append(f"rank {rank}: invalid source preflight phase")
|
||||
continue
|
||||
error = item["error"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid source preflight error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
continue
|
||||
computed = item["digest"]
|
||||
claimed = item["declared_digest"]
|
||||
if (
|
||||
not isinstance(computed, str)
|
||||
or not computed
|
||||
or not isinstance(claimed, str)
|
||||
or not claimed
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid source digest")
|
||||
continue
|
||||
agreed.append((rank, computed, claimed))
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
|
||||
)
|
||||
|
||||
canonical = agreed[0][1]
|
||||
for rank, computed, claimed in agreed:
|
||||
if computed != claimed:
|
||||
errors.append(
|
||||
f"rank {rank}: computed digest {computed} does not match "
|
||||
f"declared digest {claimed}"
|
||||
)
|
||||
if computed != canonical:
|
||||
errors.append(f"rank {rank}: source digest disagrees with rank 0")
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
|
||||
)
|
||||
return canonical
|
||||
|
||||
|
||||
def check_complete_plan_agreement(
|
||||
group: "StatelessProcessGroup",
|
||||
source_digest: str,
|
||||
dst_mesh: M2NMesh,
|
||||
dst_placements: Sequence[Placements | None],
|
||||
local_error: str | None = None,
|
||||
) -> str:
|
||||
"""Require every rank to accept the same complete transfer plan."""
|
||||
prepared_error = local_error
|
||||
digest: str | None = None
|
||||
if prepared_error is None:
|
||||
try:
|
||||
digest = complete_plan_digest(source_digest, dst_mesh, dst_placements)
|
||||
except Exception as exc:
|
||||
prepared_error = f"{type(exc).__name__}: {exc}"
|
||||
elif not isinstance(prepared_error, str) or not prepared_error:
|
||||
prepared_error = "TypeError: local plan error must be a non-empty string"
|
||||
|
||||
gathered = group.all_gather_obj(
|
||||
{"phase": "complete_plan", "digest": digest, "error": prepared_error}
|
||||
)
|
||||
errors: list[str] = []
|
||||
valid: list[tuple[int, str]] = []
|
||||
expected_fields = {"phase", "digest", "error"}
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
for rank, item in enumerate(gathered):
|
||||
if (
|
||||
not isinstance(item, Mapping)
|
||||
or set(item) != expected_fields
|
||||
or item.get("phase") != "complete_plan"
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid complete-plan envelope")
|
||||
continue
|
||||
error = item["error"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid complete-plan error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
continue
|
||||
rank_digest = item["digest"]
|
||||
if not isinstance(rank_digest, str) or not rank_digest:
|
||||
errors.append(f"rank {rank}: invalid complete-plan digest")
|
||||
continue
|
||||
valid.append((rank, rank_digest))
|
||||
if not errors:
|
||||
canonical = valid[0][1]
|
||||
for rank, rank_digest in valid[1:]:
|
||||
if rank_digest != canonical:
|
||||
errors.append(
|
||||
f"rank {rank}: complete-plan digest disagrees with rank 0"
|
||||
)
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n complete-plan preflight rejected before first reshard: "
|
||||
+ "; ".join(errors)
|
||||
)
|
||||
return valid[0][1]
|
||||
|
||||
|
||||
def check_runtime_ready_agreement(
|
||||
group: "StatelessProcessGroup", local_error: str | None
|
||||
) -> None:
|
||||
"""Require every rank to prepare local resources before NCCL init."""
|
||||
prepared_error = local_error
|
||||
if prepared_error is not None and (
|
||||
not isinstance(prepared_error, str) or not prepared_error
|
||||
):
|
||||
prepared_error = "TypeError: local runtime error must be a non-empty string"
|
||||
gathered = group.all_gather_obj({"phase": "runtime_ready", "error": prepared_error})
|
||||
errors: list[str] = []
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
for rank, item in enumerate(gathered):
|
||||
if (
|
||||
not isinstance(item, Mapping)
|
||||
or set(item) != {"phase", "error"}
|
||||
or item.get("phase") != "runtime_ready"
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid runtime readiness envelope")
|
||||
continue
|
||||
error = item.get("error")
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid runtime readiness error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n local runtime preflight failed before NCCL init: "
|
||||
+ "; ".join(errors)
|
||||
)
|
||||
|
||||
|
||||
def check_data_plane_agreement(
|
||||
group: "StatelessProcessGroup",
|
||||
unique_id_bytes: bytes | None,
|
||||
local_error: str | None = None,
|
||||
) -> None:
|
||||
"""Agree on the NCCL rendezvous identity before NCCL init."""
|
||||
prepared_error = local_error
|
||||
uid_digest: str | None = None
|
||||
if prepared_error is None:
|
||||
try:
|
||||
if unique_id_bytes is not None:
|
||||
if not isinstance(unique_id_bytes, bytes):
|
||||
raise TypeError("NCCL unique id must decode to bytes")
|
||||
uid_digest = hashlib.sha256(unique_id_bytes).hexdigest()
|
||||
except Exception as exc:
|
||||
prepared_error = f"{type(exc).__name__}: {exc}"
|
||||
elif not isinstance(prepared_error, str) or not prepared_error:
|
||||
prepared_error = "TypeError: local data-plane error must be non-empty"
|
||||
|
||||
gathered = group.all_gather_obj(
|
||||
{
|
||||
"phase": "data_plane",
|
||||
"mode": "uid" if unique_id_bytes is not None else "tcp",
|
||||
"uid_digest": uid_digest,
|
||||
"error": prepared_error,
|
||||
}
|
||||
)
|
||||
expected_fields = {
|
||||
"phase",
|
||||
"mode",
|
||||
"uid_digest",
|
||||
"error",
|
||||
}
|
||||
errors: list[str] = []
|
||||
valid: list[tuple[int, str, str | None]] = []
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
for rank, item in enumerate(gathered):
|
||||
if (
|
||||
not isinstance(item, Mapping)
|
||||
or set(item) != expected_fields
|
||||
or item.get("phase") != "data_plane"
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid data-plane preflight envelope")
|
||||
continue
|
||||
error = item["error"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid data-plane preflight error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
continue
|
||||
mode = item["mode"]
|
||||
uid = item["uid_digest"]
|
||||
if mode not in {"tcp", "uid"} or (mode == "uid") != isinstance(uid, str):
|
||||
errors.append(f"rank {rank}: invalid data-plane identity")
|
||||
continue
|
||||
valid.append((rank, mode, uid))
|
||||
if not errors:
|
||||
_, mode, uid = valid[0]
|
||||
for rank, other_mode, other_uid in valid[1:]:
|
||||
if (other_mode, other_uid) != (mode, uid):
|
||||
errors.append(f"rank {rank}: NCCL data-plane identity disagrees")
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n data-plane preflight rejected before NCCL init: "
|
||||
+ "; ".join(errors)
|
||||
)
|
||||
|
||||
|
||||
def validate_local_buffer(m2n: Any, meta: M2NParamMeta, buffer: Any) -> None:
|
||||
"""Check source dtype/shape while delegating buffer validity to M2N."""
|
||||
# A raw pointer carries no inspectable metadata. The source owns its
|
||||
# correctness; _send supplies the declared dtype and local shape to M2N.
|
||||
if isinstance(buffer, int):
|
||||
return
|
||||
|
||||
mesh, placements = resolve_layout(
|
||||
meta.source_layout.mesh, meta.source_layout.placements
|
||||
)
|
||||
actual = m2n.DistTensor(
|
||||
buffer,
|
||||
mesh=to_mesh(m2n, mesh),
|
||||
placements=to_placements(m2n, placements),
|
||||
)
|
||||
declared = m2n.DistTensor(
|
||||
None,
|
||||
local_shape=actual.local_shape,
|
||||
dtype=meta.dtype,
|
||||
mesh=to_mesh(m2n, mesh),
|
||||
placements=to_placements(m2n, placements),
|
||||
)
|
||||
if actual.dtype != declared.dtype:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source returned dtype {actual.dtype}, "
|
||||
f"but its metadata declared {declared.dtype}"
|
||||
)
|
||||
|
||||
expected = expected_local_shape(meta)
|
||||
if actual.local_shape != expected:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source returned local shape "
|
||||
f"{actual.local_shape}, but layout {meta.source_layout} implies "
|
||||
f"{expected} from global shape {meta.shape}"
|
||||
)
|
||||
|
||||
|
||||
def to_mesh(m2n: Any, mesh: M2NMesh) -> Any:
|
||||
return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank)
|
||||
|
||||
|
||||
def to_placements(m2n: Any, placements: Placements) -> list[Any]:
|
||||
return [
|
||||
m2n.Replicate() if code == REPLICATE else m2n.Shard(code) for code in placements
|
||||
]
|
||||
|
||||
|
||||
def publish_destination_placements(
|
||||
comm: Any,
|
||||
device: torch.device,
|
||||
stream: torch.cuda.Stream,
|
||||
first_worker_rank: int,
|
||||
placements: "Sequence[Placements | None] | None",
|
||||
num_parameters: int,
|
||||
) -> list[Placements | None]:
|
||||
"""Share the worker-side destination plan with every rank in the group.
|
||||
|
||||
The trainer must issue each reshard with the same destination the workers
|
||||
use, but once destinations are per-parameter they depend on the inference
|
||||
model, which only the workers can see. The first worker publishes them here,
|
||||
over the shared NCCL communicator; trainer ranks pass `None` and receive
|
||||
them, and the other workers pass their own so a disagreement is caught
|
||||
rather than deadlocking later.
|
||||
|
||||
Only placements travel: the destination *mesh* is still derived from the
|
||||
rank split, identically on both sides. The plan is published on the data
|
||||
plane before any M2N reshard collective.
|
||||
"""
|
||||
replicated_sentinel = REPLICATE - 1
|
||||
encoded = torch.full(
|
||||
(num_parameters, MESH_NDIMS),
|
||||
replicated_sentinel,
|
||||
dtype=torch.int8,
|
||||
device=device,
|
||||
)
|
||||
if comm.rank == first_worker_rank:
|
||||
if placements is None or len(placements) != num_parameters:
|
||||
raise ValueError(
|
||||
f"publishing rank needs {num_parameters} destination placements"
|
||||
)
|
||||
rows = [
|
||||
[replicated_sentinel] * MESH_NDIMS if placement is REPLICATED else placement
|
||||
for placement in placements
|
||||
]
|
||||
if rows:
|
||||
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=device))
|
||||
|
||||
comm.broadcast(encoded, encoded, root=first_worker_rank, stream=stream)
|
||||
return [
|
||||
REPLICATED
|
||||
if all(code == replicated_sentinel for code in row)
|
||||
else cast(Placements, tuple(row))
|
||||
for row in encoded.tolist()
|
||||
]
|
||||
@@ -1,599 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-side weight transfer engine built on NCCL M2N (`nccl_m2n`).
|
||||
|
||||
The trainer and the inference workers share one NCCL communicator: trainer ranks
|
||||
occupy `[0, T)`, workers `[T, T + N)`. Each parameter is moved with a single
|
||||
`nccl.m2n.reshard`, which redistributes it from the trainer's layout (FSDP / EP
|
||||
/ arbitrary DTensor sharding) to the inference layout — so the trainer sends its
|
||||
local shards and never all-gathers a full tensor, which is what the broadcast
|
||||
NCCL backend forces it to do.
|
||||
|
||||
When an incoming checkpoint parameter maps directly to a live vLLM parameter,
|
||||
M2N reshards into that worker's local model storage. Parameters whose names or
|
||||
layouts cannot be resolved receive a full tensor and fall back to
|
||||
`load_weights`, preserving the existing backend's loading behavior.
|
||||
|
||||
Both meshes participate in every reshard, in the same order, so this backend has
|
||||
the same concurrency shape as the broadcast NCCL backend: the worker must be
|
||||
inside `receive_weights` while the trainer is sending. Driving that from the
|
||||
trainer is the trainer engine's job.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
DESTINATION_SHARD_AXIS,
|
||||
M2N_WIRE_SCHEMA_VERSION,
|
||||
MESH_NDIMS,
|
||||
M2NLayout,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
M2NWireParam,
|
||||
Placements,
|
||||
check_complete_plan_agreement,
|
||||
check_data_plane_agreement,
|
||||
check_runtime_ready_agreement,
|
||||
check_source_plan_agreement,
|
||||
check_transferable,
|
||||
create_m2n_communicator,
|
||||
expected_local_shape,
|
||||
import_m2n,
|
||||
prepare_m2n_local_runtime,
|
||||
publish_destination_placements,
|
||||
resolve_layout,
|
||||
source_plan_digest,
|
||||
to_mesh,
|
||||
to_placements,
|
||||
validate_transfer_plan,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_layout import (
|
||||
M2NDestination,
|
||||
resolve_parameter_destinations,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
decode_nccl_unique_id,
|
||||
worker_init_metadata_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"M2NWeightTransferInitInfo",
|
||||
"M2NWeightTransferUpdateInfo",
|
||||
"M2NWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wire types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class M2NWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side init info: the rendezvous plus the full transfer plan.
|
||||
|
||||
Layouts are static for the whole run, so they ride the one-time init
|
||||
handshake and the per-round update info stays a list of names. Everything is
|
||||
plain JSON so the HTTP control plane carries it unchanged.
|
||||
"""
|
||||
|
||||
schema_version: int
|
||||
master_address: str
|
||||
master_port: int
|
||||
rank_offset: int
|
||||
"""First worker rank, i.e. the number of trainer ranks."""
|
||||
world_size: int
|
||||
"""Trainer ranks + all inference workers."""
|
||||
dst_mesh_dims: list[int]
|
||||
"""The inference mesh, starting at `rank_offset`. Declared rather than
|
||||
derived so both sides describe the destination identically."""
|
||||
source_digest: str
|
||||
"""SHA-256 of the ordered, versioned source parameter plan."""
|
||||
params: list[dict[str, Any]]
|
||||
"""Ordered per-parameter source metadata and layouts."""
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
max_cta: int | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.master_address, str) or not self.master_address:
|
||||
raise ValueError("`master_address` must name the metadata rendezvous")
|
||||
if (
|
||||
not isinstance(self.master_port, int)
|
||||
or isinstance(self.master_port, bool)
|
||||
or not 0 < self.master_port < 65536
|
||||
):
|
||||
raise ValueError("`master_port` must be a valid metadata port")
|
||||
if (
|
||||
not isinstance(self.rank_offset, int)
|
||||
or isinstance(self.rank_offset, bool)
|
||||
or not isinstance(self.world_size, int)
|
||||
or isinstance(self.world_size, bool)
|
||||
or self.rank_offset < 1
|
||||
or self.rank_offset >= self.world_size
|
||||
):
|
||||
raise ValueError(
|
||||
f"`rank_offset` ({self.rank_offset}) must leave at least one "
|
||||
f"trainer rank and one worker in world_size {self.world_size}"
|
||||
)
|
||||
num_workers = self.world_size - self.rank_offset
|
||||
dst = tuple(self.dst_mesh_dims)
|
||||
try:
|
||||
dst_mesh = M2NMesh(cast(tuple[int, int], dst), self.rank_offset)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"invalid `dst_mesh_dims` {dst}: {exc}") from exc
|
||||
if dst_mesh.size != num_workers:
|
||||
raise ValueError(
|
||||
f"`dst_mesh_dims` {dst} must be {MESH_NDIMS} dims covering the "
|
||||
f"{num_workers} inference workers"
|
||||
)
|
||||
|
||||
def parse_wire_params(self) -> tuple[M2NWireParam, ...]:
|
||||
"""Parse and authenticate the ordered source plan."""
|
||||
if self.schema_version != M2N_WIRE_SCHEMA_VERSION:
|
||||
raise ValueError(
|
||||
"unsupported nccl_m2n init schema version "
|
||||
f"{self.schema_version}; expected {M2N_WIRE_SCHEMA_VERSION}"
|
||||
)
|
||||
if not isinstance(self.params, list):
|
||||
raise TypeError("`params` must be a list of wire parameter mappings")
|
||||
if not isinstance(self.source_digest, str) or not self.source_digest:
|
||||
raise ValueError("`source_digest` must be a non-empty string")
|
||||
wire_params: list[M2NWireParam] = []
|
||||
names: set[str] = set()
|
||||
for index, value in enumerate(self.params):
|
||||
if not isinstance(value, Mapping):
|
||||
raise TypeError(
|
||||
f"`params[{index}]` must be a mapping, got {type(value).__name__}"
|
||||
)
|
||||
param = M2NWireParam.from_dict(value)
|
||||
if param.name in names:
|
||||
raise ValueError(
|
||||
f"nccl_m2n init params contain duplicate parameter '{param.name}'"
|
||||
)
|
||||
names.add(param.name)
|
||||
wire_params.append(param)
|
||||
parsed = tuple(wire_params)
|
||||
computed_digest = source_plan_digest(parsed)
|
||||
if self.source_digest != computed_digest:
|
||||
raise ValueError(
|
||||
"nccl_m2n init source digest does not match params: "
|
||||
f"computed {computed_digest}, declared {self.source_digest}"
|
||||
)
|
||||
return parsed
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
if self.nccl_unique_id_b64 is None:
|
||||
return None
|
||||
return decode_nccl_unique_id(
|
||||
master_address=None,
|
||||
master_port=None,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="M2NWeightTransferInitInfo data plane",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class M2NWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Per-round update info: which parameters this chunk carries, in order.
|
||||
|
||||
Shapes, dtypes and layouts were fixed at init, so a round only needs to say
|
||||
what is coming and in what order — both sides must issue their reshards in
|
||||
exactly that order.
|
||||
"""
|
||||
|
||||
names: list[str]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker engine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class M2NWeightTransferEngine(
|
||||
WeightTransferEngine[M2NWeightTransferInitInfo, M2NWeightTransferUpdateInfo]
|
||||
):
|
||||
"""Inference-side engine: receives each parameter with one reshard.
|
||||
|
||||
Resolvable parameters are resharded from the trainer layout directly into
|
||||
each worker's live local model storage. Unresolvable parameters receive a
|
||||
full tensor and use `load_weights`. In both cases, the trainer sends its
|
||||
local shards without materializing a full tensor.
|
||||
"""
|
||||
|
||||
init_info_cls = M2NWeightTransferInitInfo
|
||||
update_info_cls = M2NWeightTransferUpdateInfo
|
||||
# The destination plan is resolved against the model this engine was built
|
||||
# for, so it cannot be retargeted at a draft model mid-flight.
|
||||
supports_draft_weight_update = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: Any = None
|
||||
self._m2n: Any = None
|
||||
self._handle: Any = None
|
||||
self._metas: list[M2NParamMeta] = []
|
||||
self._dst_mesh: M2NMesh | None = None
|
||||
self._parameter_destinations: list[M2NDestination] = []
|
||||
self._index: dict[str, int] = {}
|
||||
self._uses_load_weights = False
|
||||
|
||||
def init_transfer_engine(self, init_info: M2NWeightTransferInitInfo) -> None:
|
||||
"""Agree on source and local readiness before constructing NCCL."""
|
||||
if self._handle is not None or self.model_update_group is not None:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n weight transfer is already initialized; call "
|
||||
"shutdown_weight_transfer_engine() before initializing again"
|
||||
)
|
||||
rendezvous_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=init_info.rank_offset,
|
||||
world_size=init_info.world_size,
|
||||
)
|
||||
metadata_group = worker_init_metadata_group(
|
||||
rendezvous_info, self.parallel_config
|
||||
)
|
||||
|
||||
wire_params: tuple[M2NWireParam, ...] = ()
|
||||
source_error: str | None = None
|
||||
try:
|
||||
self._m2n = import_m2n()
|
||||
wire_params = init_info.parse_wire_params()
|
||||
self._prepare_source_plan(wire_params, init_info.rank_offset)
|
||||
except Exception as exc:
|
||||
source_error = f"{type(exc).__name__}: {exc}"
|
||||
self._metas = []
|
||||
check_source_plan_agreement(
|
||||
metadata_group,
|
||||
wire_params,
|
||||
expected_digest=init_info.source_digest,
|
||||
local_error=source_error,
|
||||
)
|
||||
|
||||
destination_error: str | None = None
|
||||
try:
|
||||
self._prepare_destination_plan(init_info)
|
||||
except Exception as exc:
|
||||
destination_error = f"{type(exc).__name__}: {exc}"
|
||||
self._parameter_destinations = []
|
||||
|
||||
unique_id_bytes: bytes | None = None
|
||||
data_plane_error: str | None = None
|
||||
try:
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
except Exception as exc:
|
||||
data_plane_error = f"{type(exc).__name__}: {exc}"
|
||||
check_data_plane_agreement(
|
||||
metadata_group,
|
||||
unique_id_bytes,
|
||||
data_plane_error,
|
||||
)
|
||||
|
||||
handle = None
|
||||
group = None
|
||||
try:
|
||||
runtime_error = destination_error
|
||||
nccl = None
|
||||
device = None
|
||||
if runtime_error is None:
|
||||
try:
|
||||
nccl, device, handle = prepare_m2n_local_runtime(
|
||||
self._m2n, init_info.max_cta
|
||||
)
|
||||
except Exception as exc:
|
||||
runtime_error = f"{type(exc).__name__}: {exc}"
|
||||
check_runtime_ready_agreement(metadata_group, runtime_error)
|
||||
assert nccl is not None and device is not None and handle is not None
|
||||
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
|
||||
|
||||
# Keep the merged destination-placement publication unchanged: source
|
||||
# agreement is independent of worker-side destination selection.
|
||||
mine = [
|
||||
destination.placements for destination in self._parameter_destinations
|
||||
]
|
||||
agreed = publish_destination_placements(
|
||||
group,
|
||||
self.device,
|
||||
torch.cuda.current_stream(),
|
||||
init_info.rank_offset,
|
||||
mine,
|
||||
len(self._metas),
|
||||
)
|
||||
complete_plan_error: str | None = None
|
||||
try:
|
||||
if agreed != mine:
|
||||
mismatched = next(
|
||||
meta.name
|
||||
for meta, ours, theirs in zip(self._metas, mine, agreed)
|
||||
if ours != theirs
|
||||
)
|
||||
raise RuntimeError(
|
||||
"inference workers resolved different nccl_m2n "
|
||||
f"destinations (first mismatch: '{mismatched}'); all "
|
||||
"workers must run the same model and parallel config"
|
||||
)
|
||||
assert self._dst_mesh is not None
|
||||
validate_transfer_plan(self._metas, self._dst_mesh, agreed)
|
||||
except Exception as exc:
|
||||
complete_plan_error = f"{type(exc).__name__}: {exc}"
|
||||
assert self._dst_mesh is not None
|
||||
check_complete_plan_agreement(
|
||||
metadata_group,
|
||||
init_info.source_digest,
|
||||
self._dst_mesh,
|
||||
agreed,
|
||||
complete_plan_error,
|
||||
)
|
||||
except BaseException:
|
||||
if group is not None:
|
||||
try:
|
||||
# Transport cleanup only; reload validity belongs to the
|
||||
# model-update lifecycle, not this backend.
|
||||
group.abort()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to abort partially initialized nccl_m2n communicator"
|
||||
)
|
||||
if handle is not None:
|
||||
try:
|
||||
handle.destroy()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to destroy partially initialized nccl_m2n handle"
|
||||
)
|
||||
raise
|
||||
|
||||
self._handle = handle
|
||||
self.model_update_group = group
|
||||
|
||||
def _prepare_source_plan(
|
||||
self,
|
||||
wire_params: tuple[M2NWireParam, ...],
|
||||
num_trainer_ranks: int,
|
||||
) -> None:
|
||||
"""Validate worker-visible source metadata without constructing NCCL."""
|
||||
self._metas = []
|
||||
for param in wire_params:
|
||||
dtype = getattr(torch, param.dtype_name, None)
|
||||
if not isinstance(dtype, torch.dtype):
|
||||
raise ValueError(
|
||||
f"parameter '{param.name}' has invalid dtype name "
|
||||
f"'{param.dtype_name}'; expected the name of a torch.dtype"
|
||||
)
|
||||
check_transferable(param.name, dtype, param.shape)
|
||||
layout = M2NLayout(M2NMesh(param.src_mesh_dims, 0), param.src_placements)
|
||||
if layout.mesh.size != num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"parameter '{param.name}' source mesh covers "
|
||||
f"{layout.mesh.size} ranks, but there are "
|
||||
f"{num_trainer_ranks} trainer ranks"
|
||||
)
|
||||
meta = M2NParamMeta(param.name, dtype, param.shape, layout)
|
||||
expected_local_shape(meta)
|
||||
self._metas.append(meta)
|
||||
|
||||
def _prepare_destination_plan(self, init_info: M2NWeightTransferInitInfo) -> None:
|
||||
"""Resolve the merged-#51520 in-place/full-fallback destination plan."""
|
||||
# The inference topology, as declared by the trainer. This is the mesh
|
||||
# a sharded destination is placed over; a replicated one is described
|
||||
# by the alternate descriptor `resolve_layout` derives from it, which
|
||||
# covers the same ranks but is not this mesh.
|
||||
self._dst_mesh = M2NMesh(
|
||||
cast(tuple[int, int], tuple(init_info.dst_mesh_dims)),
|
||||
init_info.rank_offset,
|
||||
)
|
||||
parallel_config = self.parallel_config
|
||||
shard_axis_size = self._dst_mesh.dims[DESTINATION_SHARD_AXIS]
|
||||
quantization = getattr(self.model_config, "quantization", None)
|
||||
# A quantized weight loader may dequantize / transpose / requantize, and
|
||||
# under PP a rank's local shape is not the checkpoint shape split over
|
||||
# the shard axis. Both break shape-derived resolution, so take the
|
||||
# fallback.
|
||||
allow_direct = (
|
||||
quantization is None and parallel_config.pipeline_parallel_size == 1
|
||||
)
|
||||
# Match every incoming checkpoint parameter to its rank-local target.
|
||||
# Resolvable parameters can be resharded directly into live model
|
||||
# storage; the rest receive a full tensor for `load_weights`.
|
||||
self._parameter_destinations = resolve_parameter_destinations(
|
||||
self.model,
|
||||
[m.name for m in self._metas],
|
||||
[m.dtype for m in self._metas],
|
||||
[m.shape for m in self._metas],
|
||||
num_workers=self._dst_mesh.size,
|
||||
shard_axis_size=shard_axis_size,
|
||||
allow_direct=allow_direct,
|
||||
)
|
||||
validate_transfer_plan(
|
||||
self._metas,
|
||||
self._dst_mesh,
|
||||
[destination.placements for destination in self._parameter_destinations],
|
||||
)
|
||||
# Update requests carry names only, so cache their plan indices. Track
|
||||
# whether any fallback entry requires the `load_weights` lifecycle.
|
||||
self._index = {meta.name: i for i, meta in enumerate(self._metas)}
|
||||
self._uses_load_weights = any(
|
||||
not destination.direct for destination in self._parameter_destinations
|
||||
)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Set up layerwise reloading, but only if some parameter needs it.
|
||||
|
||||
Directly-resharded parameters are written in place and never go through
|
||||
`load_weights`, so a plan with no fallback entries has nothing to
|
||||
reload.
|
||||
"""
|
||||
if not self._uses_load_weights:
|
||||
return
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading when the plan uses fallback entries."""
|
||||
if not self._uses_load_weights:
|
||||
return
|
||||
from vllm.model_executor.model_loader.reload import finalize_layerwise_reload
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: M2NWeightTransferUpdateInfo) -> None:
|
||||
"""Receive each requested parameter using its initialization-time plan."""
|
||||
if self._handle is None or self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
if not isinstance(update_info.names, list) or any(
|
||||
not isinstance(name, str) for name in update_info.names
|
||||
):
|
||||
raise TypeError("nccl_m2n update names must be a list of strings")
|
||||
|
||||
requested = []
|
||||
for name in update_info.names:
|
||||
index = self._index.get(name)
|
||||
if index is None:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' was not declared at init; the "
|
||||
"trainer must send the same parameter set it announced"
|
||||
)
|
||||
requested.append((name, index))
|
||||
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
disable_mtp_completeness_check,
|
||||
)
|
||||
|
||||
stream = torch.cuda.current_stream()
|
||||
|
||||
# Reshard lazily so one `load_weights` invocation sees the complete
|
||||
# fallback sequence without staging every full tensor at once. Direct
|
||||
# destinations still execute in request order as the iterator advances.
|
||||
def reshard_requested_weights() -> Iterator[tuple[str, torch.Tensor]]:
|
||||
for name, index in requested:
|
||||
meta = self._metas[index]
|
||||
destination = self._parameter_destinations[index]
|
||||
buffer = destination.tensor
|
||||
if buffer is None:
|
||||
buffer = torch.empty(
|
||||
meta.shape, dtype=meta.dtype, device=self.device
|
||||
)
|
||||
|
||||
self._reshard(
|
||||
self.model_update_group,
|
||||
stream,
|
||||
meta,
|
||||
destination.placements,
|
||||
buffer,
|
||||
)
|
||||
|
||||
if destination.tensor is None:
|
||||
# `load_weights` reads on the host stream, so the transfer
|
||||
# has to have landed before it runs.
|
||||
stream.synchronize()
|
||||
yield name, buffer
|
||||
|
||||
has_fallback = any(
|
||||
self._parameter_destinations[index].tensor is None for _, index in requested
|
||||
)
|
||||
with disable_mtp_completeness_check():
|
||||
received_weights = reshard_requested_weights()
|
||||
if has_fallback:
|
||||
# Some model loaders keep invocation-local state to combine
|
||||
# related checkpoint tensors, so preserve one iterable scope.
|
||||
self.model.load_weights(received_weights)
|
||||
else:
|
||||
# The iterator performs the reshards. With nothing to yield to
|
||||
# `load_weights`, exhaust it here to execute direct transfers.
|
||||
for _ in received_weights:
|
||||
pass
|
||||
|
||||
def _reshard(
|
||||
self,
|
||||
comm: Any,
|
||||
stream: torch.cuda.Stream,
|
||||
src_meta: M2NParamMeta,
|
||||
dst_placements: Placements | None,
|
||||
dst_buffer: torch.Tensor,
|
||||
) -> None:
|
||||
"""Reshard one parameter from its trainer layout into a worker buffer.
|
||||
|
||||
`dst_placements` describes the buffer's logical placement over the
|
||||
worker mesh; `dst_buffer` is this rank's physical storage.
|
||||
"""
|
||||
assert self._dst_mesh is not None
|
||||
m2n = self._m2n
|
||||
src_mesh, resolved_src_placements = resolve_layout(
|
||||
src_meta.source_layout.mesh, src_meta.source_layout.placements
|
||||
)
|
||||
dst_mesh, resolved_dst_placements = resolve_layout(
|
||||
self._dst_mesh, dst_placements
|
||||
)
|
||||
m2n.reshard(
|
||||
None,
|
||||
dst_buffer,
|
||||
comm,
|
||||
stream,
|
||||
src_mesh=to_mesh(m2n, src_mesh),
|
||||
src_placements=to_placements(m2n, resolved_src_placements),
|
||||
dst_mesh=to_mesh(m2n, dst_mesh),
|
||||
dst_placements=to_placements(m2n, resolved_dst_placements),
|
||||
handle=self._handle,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Finish pending GPU work and release M2N communication state."""
|
||||
handle, self._handle = self._handle, None
|
||||
group, self.model_update_group = self.model_update_group, None
|
||||
self._m2n = None
|
||||
self._metas = []
|
||||
self._dst_mesh = None
|
||||
self._parameter_destinations = []
|
||||
self._index = {}
|
||||
self._uses_load_weights = False
|
||||
|
||||
first_error: BaseException | None = None
|
||||
|
||||
def attempt(action: Any, message: str) -> None:
|
||||
nonlocal first_error
|
||||
try:
|
||||
action()
|
||||
except BaseException as exc:
|
||||
if first_error is None:
|
||||
first_error = exc
|
||||
logger.exception(message)
|
||||
|
||||
if handle is not None or group is not None:
|
||||
# M2N does not synchronize caller streams on finalize.
|
||||
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
|
||||
if handle is not None:
|
||||
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
|
||||
if group is not None:
|
||||
attempt(group.abort, "failed to abort nccl_m2n communicator")
|
||||
if first_error is not None:
|
||||
raise first_error
|
||||
@@ -1,261 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Destination-layout resolution for the NCCL M2N weight transfer backend.
|
||||
|
||||
For every incoming checkpoint parameter the worker needs a destination buffer
|
||||
plus its placement over the inference mesh. Two outcomes are possible:
|
||||
|
||||
* **direct** — when the model is not quantized, the parameter maps 1:1 onto a
|
||||
live vLLM parameter and its loader is a small, known-safe copy or tensor-
|
||||
parallel loader. The loader's declared input/output dimension determines the
|
||||
placement; shapes are validation, not an inference mechanism. The reshard
|
||||
writes straight into the live parameter, so each rank receives only its own
|
||||
shard and nothing is copied afterwards.
|
||||
* **fallback** — anything else. The reshard delivers the whole tensor to every
|
||||
rank and `load_weights` does the sharding, exactly as the broadcast NCCL
|
||||
backend does. Fused parameters (`qkv_proj`, `gate_up_proj`, MoE `w13`/`w2`)
|
||||
take this path: the checkpoint name does not name a vLLM parameter, so there
|
||||
is nothing to resolve against.
|
||||
|
||||
Correctness never depends on a parameter resolving — the fallback is always
|
||||
available and is the same path the existing backend uses.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
DESTINATION_REPLICA_AXIS,
|
||||
DESTINATION_SHARD_AXIS,
|
||||
MESH_NDIMS,
|
||||
REPLICATE,
|
||||
REPLICATED,
|
||||
Placements,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_DIRECT_WEIGHT_LOADING_BLOCKLIST = {
|
||||
"vllm.model_executor.models.gpt2.GPT2Model",
|
||||
}
|
||||
|
||||
|
||||
def _module_type_name(module: torch.nn.Module) -> str:
|
||||
cls = type(module)
|
||||
return f"{cls.__module__}.{cls.__name__}"
|
||||
|
||||
|
||||
def _destination_placements(shard_dim: int) -> Placements:
|
||||
"""Place one tensor shard on the configured destination mesh axis."""
|
||||
placements = [REPLICATE] * MESH_NDIMS
|
||||
placements[DESTINATION_REPLICA_AXIS] = REPLICATE
|
||||
placements[DESTINATION_SHARD_AXIS] = shard_dim
|
||||
return cast(Placements, tuple(placements))
|
||||
|
||||
|
||||
@dataclass
|
||||
class M2NDestination:
|
||||
"""Where one checkpoint parameter lands on this worker."""
|
||||
|
||||
name: str
|
||||
placements: Placements | None
|
||||
"""Placement over the inference mesh, or `REPLICATED` for the fallback."""
|
||||
tensor: torch.Tensor | None
|
||||
"""The live parameter view to reshard into, or None for the fallback path
|
||||
(the engine allocates a full replica per round and calls `load_weights`)."""
|
||||
|
||||
@property
|
||||
def direct(self) -> bool:
|
||||
return self.tensor is not None
|
||||
|
||||
|
||||
def _base_loader_shard_dim(
|
||||
param: torch.nn.Parameter,
|
||||
shard_axis_size: int,
|
||||
) -> int | None:
|
||||
"""Return a known-safe loader's declared shard dim, or reject it.
|
||||
|
||||
Loader identity is deliberately fail-closed. A missing loader is not
|
||||
treated as ``default_weight_loader`` because a model-level ``load_weights``
|
||||
implementation may transform the tensor before it reaches that default.
|
||||
Wrappers and partials are also rejected: recognizing their wrapped callable
|
||||
would ignore behavior added by the wrapper.
|
||||
"""
|
||||
loader = getattr(param, "weight_loader", None)
|
||||
if loader is None or isinstance(loader, partial):
|
||||
return None
|
||||
|
||||
owner = loader.__self__ if inspect.ismethod(loader) else None
|
||||
loader_fn = loader.__func__ if inspect.ismethod(loader) else loader
|
||||
|
||||
# Lazy imports avoid making the weight-transfer registry import all model
|
||||
# executor layers during normal vLLM startup.
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
replicated_loaders = {
|
||||
default_weight_loader,
|
||||
ReplicatedLinear.weight_loader,
|
||||
}
|
||||
column_loaders = {
|
||||
ColumnParallelLinear.weight_loader,
|
||||
ColumnParallelLinear.weight_loader_v2,
|
||||
}
|
||||
row_loaders = {
|
||||
RowParallelLinear.weight_loader,
|
||||
RowParallelLinear.weight_loader_v2,
|
||||
}
|
||||
|
||||
# These attributes signal packing, a transform, or pre-sharded checkpoint
|
||||
# storage. Such semantics cannot be reproduced by a plain M2N placement.
|
||||
# Check them before accepting replicated loaders too: model-level loading
|
||||
# may transform a tensor before handing it to an otherwise plain copier.
|
||||
metadata_attrs = ("packed_dim", "packed_factor", "marlin_tile_size")
|
||||
flag_attrs = ("needs_scalar_to_array", "is_sharded_weight", "is_transposed")
|
||||
if any(getattr(param, attr, None) is not None for attr in metadata_attrs):
|
||||
return None
|
||||
if any(bool(getattr(param, attr, False)) for attr in flag_attrs):
|
||||
return None
|
||||
|
||||
if loader_fn in replicated_loaders:
|
||||
return REPLICATE
|
||||
if loader_fn not in column_loaders | row_loaders:
|
||||
return None
|
||||
|
||||
if owner is None or getattr(owner, "tp_size", None) != shard_axis_size:
|
||||
return None
|
||||
|
||||
attr = "output_dim" if loader_fn in column_loaders else "input_dim"
|
||||
dim = getattr(param, attr, None)
|
||||
return dim if isinstance(dim, int) and dim >= 0 else None
|
||||
|
||||
|
||||
def _validated_shard_dim(
|
||||
global_shape: Sequence[int],
|
||||
local_shape: Sequence[int],
|
||||
declared_dim: int,
|
||||
shard_axis_size: int,
|
||||
) -> int | None:
|
||||
"""Validate a loader-declared shard dimension against both shapes."""
|
||||
if len(global_shape) != len(local_shape):
|
||||
return None
|
||||
|
||||
if declared_dim == REPLICATE:
|
||||
return REPLICATE if tuple(global_shape) == tuple(local_shape) else None
|
||||
if declared_dim >= len(global_shape):
|
||||
return None
|
||||
for dim, (whole, local) in enumerate(zip(global_shape, local_shape)):
|
||||
expected = local * shard_axis_size if dim == declared_dim else local
|
||||
if whole != expected:
|
||||
return None
|
||||
return declared_dim
|
||||
|
||||
|
||||
def resolve_parameter_destinations(
|
||||
model: torch.nn.Module,
|
||||
names: Sequence[str],
|
||||
dtypes: Sequence[torch.dtype],
|
||||
shapes: Sequence[Sequence[int]],
|
||||
*,
|
||||
num_workers: int,
|
||||
shard_axis_size: int,
|
||||
allow_direct: bool,
|
||||
) -> list[M2NDestination]:
|
||||
"""Build this worker's destination plan, one entry per checkpoint parameter.
|
||||
|
||||
Placements are relative to the inference mesh: axis 0 replicates and axis 1
|
||||
shards. `allow_direct=False` forces every parameter onto the fallback path;
|
||||
the engine sets it for pipeline-parallel or quantized deployments, where a
|
||||
parameter's local shape is not simply the checkpoint shape split across the
|
||||
shard axis. Parameters below a model type with known model-level checkpoint
|
||||
preprocessing also fall back, since bypassing `load_weights` would skip the
|
||||
transformation.
|
||||
"""
|
||||
params = dict(model.named_parameters()) if allow_direct else {}
|
||||
blocked_param_ids = {
|
||||
id(param)
|
||||
for module in model.modules()
|
||||
if _module_type_name(module) in _DIRECT_WEIGHT_LOADING_BLOCKLIST
|
||||
for param in module.parameters()
|
||||
}
|
||||
|
||||
destinations: list[M2NDestination] = []
|
||||
for name, dtype, shape in zip(names, dtypes, shapes):
|
||||
param = params.get(name)
|
||||
dim = None
|
||||
if (
|
||||
param is not None
|
||||
and id(param) not in blocked_param_ids
|
||||
and param.dtype == dtype
|
||||
and param.data.is_contiguous()
|
||||
and num_workers % shard_axis_size == 0
|
||||
):
|
||||
declared_dim = _base_loader_shard_dim(param, shard_axis_size)
|
||||
if declared_dim is not None:
|
||||
dim = _validated_shard_dim(
|
||||
shape, param.shape, declared_dim, shard_axis_size
|
||||
)
|
||||
|
||||
if dim is None:
|
||||
# A replicated parameter is identical on every rank, so it needs no
|
||||
# placement of its own — REPLICATED lets resolve_layout spread it
|
||||
# over all the workers regardless of how the mesh is factored.
|
||||
destinations.append(M2NDestination(name, REPLICATED, None))
|
||||
elif dim == REPLICATE:
|
||||
assert param is not None
|
||||
destinations.append(M2NDestination(name, REPLICATED, param.data))
|
||||
else:
|
||||
assert param is not None
|
||||
destinations.append(
|
||||
M2NDestination(name, _destination_placements(dim), param.data)
|
||||
)
|
||||
|
||||
# Layerwise reload finalizes a module as a unit. Mixing direct and fallback
|
||||
# parameters within one module can copy stale storage over the direct load.
|
||||
fallback_modules = {
|
||||
name.rpartition(".")[0]
|
||||
for name, destination in zip(names, destinations)
|
||||
if not destination.direct and name in params
|
||||
}
|
||||
destinations = [
|
||||
M2NDestination(destination.name, REPLICATED, None)
|
||||
if destination.direct
|
||||
and destination.name.rpartition(".")[0] in fallback_modules
|
||||
else destination
|
||||
for destination in destinations
|
||||
]
|
||||
|
||||
num_direct = sum(d.direct for d in destinations)
|
||||
parameter_bytes = [
|
||||
math.prod(shape) * dtype.itemsize for dtype, shape in zip(dtypes, shapes)
|
||||
]
|
||||
total_bytes = sum(parameter_bytes)
|
||||
direct_bytes = sum(
|
||||
nbytes
|
||||
for destination, nbytes in zip(destinations, parameter_bytes)
|
||||
if destination.direct
|
||||
)
|
||||
direct_byte_percentage = 100 * direct_bytes / total_bytes if total_bytes else 0
|
||||
logger.info(
|
||||
"nccl_m2n destination plan: %d/%d parameters resharded directly into "
|
||||
"the model, %d via full-tensor fallback; direct byte coverage: "
|
||||
"%d/%d bytes (%.1f%%)",
|
||||
num_direct,
|
||||
len(destinations),
|
||||
len(destinations) - num_direct,
|
||||
direct_bytes,
|
||||
total_bytes,
|
||||
direct_byte_percentage,
|
||||
)
|
||||
return destinations
|
||||
@@ -1,207 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Trainer-side weight sources for the NCCL M2N backend.
|
||||
|
||||
m2n plans a transfer from both sides' layouts, so the trainer has to say how it
|
||||
holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not
|
||||
express. This module supplies the m2n flavor of both, and the DTensor-backed
|
||||
implementation that covers the common trainer. Each parameter owns its source
|
||||
mesh and placements because dense and expert weights can use different
|
||||
factorizations over the same trainer ranks.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import WeightSource
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
REPLICATE,
|
||||
REPLICATED,
|
||||
M2NLayout,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
Placements,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"M2NWeightSource",
|
||||
"M2NManifestEntry",
|
||||
"ManifestM2NWeightSource",
|
||||
"DTensorModuleSource",
|
||||
"mesh_from_tensor",
|
||||
"placements_from_tensor",
|
||||
]
|
||||
|
||||
|
||||
class M2NWeightSource(WeightSource):
|
||||
"""A `WeightSource` that also describes how the trainer holds its weights.
|
||||
|
||||
Unlike `ModuleSource`, iteration yields each rank's **local shard**, not a
|
||||
materialized full tensor: gathering is exactly the cost m2n removes.
|
||||
"""
|
||||
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Name, dtype, full shape, and source layout for each parameter."""
|
||||
raise NotImplementedError
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
|
||||
"""Yield `(name, local buffer)` pairs in metadata order.
|
||||
|
||||
Providers must enqueue materialization on the caller's current CUDA
|
||||
stream, or return a buffer whose writes are already visible to it.
|
||||
Buffers may be torch tensors, CUDA Array Interface objects, or raw
|
||||
device pointers accepted by nccl4py. Providers own correct
|
||||
communicator-device placement. Raw-pointer providers additionally own
|
||||
allocation lifetime, dtype, and shape correctness.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NManifestEntry:
|
||||
"""One immutable source descriptor with a lazy rank-local tensor view."""
|
||||
|
||||
key: str
|
||||
dtype: torch.dtype
|
||||
global_shape: tuple[int, ...]
|
||||
source_layout: M2NLayout
|
||||
local_tensor: Callable[[], Any]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "global_shape", tuple(self.global_shape))
|
||||
if not self.key:
|
||||
raise ValueError("M2N manifest key must not be empty")
|
||||
if not isinstance(self.dtype, torch.dtype):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' has invalid dtype {self.dtype!r}"
|
||||
)
|
||||
if not isinstance(self.source_layout, M2NLayout):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' source_layout must be an M2NLayout"
|
||||
)
|
||||
if not callable(self.local_tensor):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' local_tensor must be callable"
|
||||
)
|
||||
|
||||
@property
|
||||
def metadata(self) -> M2NParamMeta:
|
||||
"""Return the serializable portion consumed during initialization."""
|
||||
return M2NParamMeta(
|
||||
self.key,
|
||||
self.dtype,
|
||||
self.global_shape,
|
||||
self.source_layout,
|
||||
)
|
||||
|
||||
|
||||
class ManifestM2NWeightSource(M2NWeightSource):
|
||||
"""An ordered M2N source backed by lazy, rank-local manifest entries."""
|
||||
|
||||
def __init__(self, entries: Sequence[M2NManifestEntry]) -> None:
|
||||
self._entries = tuple(entries)
|
||||
self._metadata = tuple(entry.metadata for entry in self._entries)
|
||||
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Return a fresh list while preserving the immutable manifest order."""
|
||||
return list(self._metadata)
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
|
||||
"""Materialize one rank-local buffer at a time."""
|
||||
for entry in self._entries:
|
||||
yield entry.key, entry.local_tensor()
|
||||
|
||||
|
||||
def _placement_code(placement: Any) -> int:
|
||||
"""Map a `torch.distributed` placement onto an m2n placement code."""
|
||||
name = type(placement).__name__
|
||||
if name == "Replicate":
|
||||
return REPLICATE
|
||||
if name == "Shard":
|
||||
return int(placement.dim)
|
||||
raise ValueError(
|
||||
f"nccl_m2n cannot express the {name} placement; only Replicate and "
|
||||
"Shard are supported"
|
||||
)
|
||||
|
||||
|
||||
def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh:
|
||||
"""The mesh a parameter lives on, as an `M2NMesh`.
|
||||
|
||||
Tensors without DeviceMesh metadata are treated as replicated. Implicitly
|
||||
sharded tensors require a custom source with an explicit layout.
|
||||
"""
|
||||
device_mesh = getattr(tensor, "device_mesh", None)
|
||||
if device_mesh is None:
|
||||
return M2NMesh((num_trainer_ranks, 1), 0)
|
||||
|
||||
grid = device_mesh.mesh
|
||||
ranks = grid.flatten().tolist()
|
||||
if ranks != list(range(len(ranks))):
|
||||
raise ValueError(
|
||||
"nccl_m2n requires the trainer's device mesh to cover the "
|
||||
f"contiguous rank interval [0, {len(ranks)}); got {ranks}"
|
||||
)
|
||||
if grid.ndim > 2:
|
||||
raise ValueError(
|
||||
f"nccl_m2n supports 1-D and 2-D device meshes, got {grid.ndim}-D"
|
||||
)
|
||||
dims = tuple(grid.shape)
|
||||
return M2NMesh(dims if len(dims) == 2 else (dims[0], 1), 0)
|
||||
|
||||
|
||||
def placements_from_tensor(tensor: torch.Tensor) -> Placements | None:
|
||||
"""How a parameter is placed over its mesh, or `REPLICATED`.
|
||||
|
||||
`REPLICATED` is returned rather than a `(REPLICATE, REPLICATE)` pair: m2n
|
||||
cannot express that, and the size-1-axis encoding it needs instead is
|
||||
applied later by `resolve_layout`, which owns that workaround.
|
||||
"""
|
||||
placements = getattr(tensor, "placements", None)
|
||||
if placements is None:
|
||||
return REPLICATED
|
||||
codes = [_placement_code(p) for p in placements]
|
||||
if all(code == REPLICATE for code in codes):
|
||||
return REPLICATED
|
||||
if len(codes) == 1:
|
||||
return (codes[0], REPLICATE)
|
||||
return (codes[0], codes[1])
|
||||
|
||||
|
||||
class DTensorModuleSource(M2NWeightSource):
|
||||
"""`M2NWeightSource` over `module.named_parameters()`.
|
||||
|
||||
Covers both the FSDP/DTensor trainer (placement read off each parameter,
|
||||
local shard yielded via `to_local()`) and the plain replicated trainer, with
|
||||
no special casing. Trainers with a custom producer (a Megatron export, MoE
|
||||
re-fusing) subclass `M2NWeightSource` instead.
|
||||
"""
|
||||
|
||||
def __init__(self, module: torch.nn.Module, num_trainer_ranks: int) -> None:
|
||||
"""Wrap `module`; `num_trainer_ranks` sizes the mesh for plain tensors."""
|
||||
self._module = module
|
||||
self._num_trainer_ranks = num_trainer_ranks
|
||||
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Read each global shape and source layout without gathering shards."""
|
||||
return [
|
||||
M2NParamMeta(
|
||||
name,
|
||||
p.dtype,
|
||||
tuple(p.shape),
|
||||
M2NLayout(
|
||||
mesh_from_tensor(p, self._num_trainer_ranks),
|
||||
placements_from_tensor(p),
|
||||
),
|
||||
)
|
||||
for name, p in self._module.named_parameters()
|
||||
]
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Yield each parameter's local shard (`to_local()`), or the tensor itself."""
|
||||
for name, param in self._module.named_parameters():
|
||||
to_local = getattr(param, "to_local", None)
|
||||
yield name, (to_local() if callable(to_local) else param)
|
||||
@@ -1,628 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Trainer-side weight transfer engine for the NCCL M2N backend.
|
||||
|
||||
Symmetric to `M2NWeightTransferEngine` but in the training process. Every
|
||||
trainer rank joins the shared communicator and runs every reshard, sending its
|
||||
own local shard; only rank 0 touches the inference control plane.
|
||||
|
||||
Workers join the communicator during init and run reshard from inside
|
||||
`update_weights`, so those two RPCs overlap with work on this side. Both run on
|
||||
a helper thread and are joined afterwards, which keeps that overlap inside the
|
||||
engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather than
|
||||
in every caller.
|
||||
"""
|
||||
|
||||
import socket
|
||||
from collections.abc import Mapping
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
M2N_WIRE_SCHEMA_VERSION,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
M2NWireParam,
|
||||
Placements,
|
||||
check_complete_plan_agreement,
|
||||
check_data_plane_agreement,
|
||||
check_runtime_ready_agreement,
|
||||
check_source_plan_agreement,
|
||||
check_transferable,
|
||||
create_m2n_communicator,
|
||||
expected_local_shape,
|
||||
import_m2n,
|
||||
prepare_m2n_local_runtime,
|
||||
publish_destination_placements,
|
||||
resolve_layout,
|
||||
source_plan_digest,
|
||||
to_mesh,
|
||||
to_placements,
|
||||
validate_local_buffer,
|
||||
validate_transfer_plan,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
decode_nccl_unique_id,
|
||||
stateless_init_metadata_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"]
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class M2NTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for nccl_m2n.
|
||||
|
||||
`rank` (from `TrainerInitInfo`) is this trainer process's rank; it is also
|
||||
its rank in the shared communicator, since the trainer occupies
|
||||
`[0, num_trainer_ranks)`. Rank 0 drives the control plane.
|
||||
"""
|
||||
|
||||
backend: ClassVar[str] = "nccl_m2n"
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
"""Trainer ranks + all inference workers."""
|
||||
num_trainer_ranks: int = 1
|
||||
dst_mesh_dims: tuple[int, int] | None = None
|
||||
"""How the inference ranks are laid out: axis 0 replicates and axis 1
|
||||
shards. The trainer declares it so both sides describe the destination
|
||||
identically; defaults to a flat `(num_workers, 1)`, which is all a
|
||||
replicated destination needs.
|
||||
|
||||
Destination meshes remain common to every parameter; per-parameter
|
||||
destination meshes are intentionally future work.
|
||||
"""
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
"""Optional UID for the NCCL data plane; TCP remains the metadata plane."""
|
||||
max_cta: int | None = None
|
||||
listen_socket: socket.socket | None = field(default=None, repr=False, compare=False)
|
||||
"""Optional rank-zero TCPStore reservation; never sent to workers."""
|
||||
|
||||
@property
|
||||
def destination_mesh_dims(self) -> tuple[int, int]:
|
||||
"""`dst_mesh_dims`, or a flat mesh over every inference worker."""
|
||||
if self.dst_mesh_dims is not None:
|
||||
return cast(tuple[int, int], tuple(self.dst_mesh_dims))
|
||||
return (self.world_size - self.num_trainer_ranks, 1)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Reject rank counts or destination meshes that cannot form a group."""
|
||||
if not isinstance(self.master_address, str) or not self.master_address:
|
||||
raise ValueError("`master_address` must name the metadata rendezvous")
|
||||
if (
|
||||
not isinstance(self.master_port, int)
|
||||
or isinstance(self.master_port, bool)
|
||||
or not 0 < self.master_port < 65536
|
||||
):
|
||||
raise ValueError("`master_port` must be a valid metadata port")
|
||||
if not 0 < self.num_trainer_ranks < self.world_size:
|
||||
raise ValueError(
|
||||
f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at "
|
||||
f"least one worker in world_size {self.world_size}"
|
||||
)
|
||||
num_workers = self.world_size - self.num_trainer_ranks
|
||||
if self.dst_mesh_dims is not None:
|
||||
dims = tuple(self.dst_mesh_dims)
|
||||
try:
|
||||
mesh = M2NMesh(cast(tuple[int, int], dims), self.num_trainer_ranks)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"invalid `dst_mesh_dims` {dims}: {exc}") from exc
|
||||
if mesh.size != num_workers:
|
||||
raise ValueError(
|
||||
f"`dst_mesh_dims` {dims} must be two dims covering the "
|
||||
f"{num_workers} inference workers"
|
||||
)
|
||||
if not 0 <= self.rank < self.num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"trainer `rank` ({self.rank}) must be below "
|
||||
f"`num_trainer_ranks` ({self.num_trainer_ranks})"
|
||||
)
|
||||
if self.listen_socket is not None:
|
||||
if self.rank != 0:
|
||||
raise ValueError("only trainer rank zero may reserve the TCPStore")
|
||||
if not isinstance(self.listen_socket, socket.socket):
|
||||
raise TypeError("listen_socket must be a socket.socket")
|
||||
if self.listen_socket.fileno() < 0:
|
||||
raise ValueError("listen_socket is already closed")
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
if self.nccl_unique_id_b64 is None:
|
||||
return None
|
||||
return decode_nccl_unique_id(
|
||||
master_address=None,
|
||||
master_port=None,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="M2NTrainerInitInfo data plane",
|
||||
)
|
||||
|
||||
|
||||
class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitInfo]):
|
||||
"""Trainer-side engine: sends every rank's local shard, once per parameter.
|
||||
|
||||
Called on every trainer rank. All ranks join the communicator and run every
|
||||
reshard; only rank 0 touches the control plane.
|
||||
"""
|
||||
|
||||
init_info_cls = M2NTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: M2NWeightSource,
|
||||
is_sender: bool,
|
||||
) -> None:
|
||||
"""Hold the client and source; `trainer_init` fills in the group."""
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.group: Any = None
|
||||
self._m2n: Any = None
|
||||
self._handle: Any = None
|
||||
self._metas: list[M2NParamMeta] = []
|
||||
self._wire_params: list[M2NWireParam] = []
|
||||
self._source_digest = ""
|
||||
self._dst_mesh: M2NMesh | None = None
|
||||
self._dst_placements: list[Placements | None] = []
|
||||
self._metadata_group: StatelessProcessGroup | None = None
|
||||
self._num_trainer_ranks = 0
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: M2NTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
"""Build the engine and rendezvous with the inference side.
|
||||
|
||||
Runs on *every* trainer rank. Rank 0 additionally ships the transfer
|
||||
plan to the workers and drives the control plane; the other ranks only
|
||||
build local state and join the communicator. Every rank participates
|
||||
in every reshard and sends its local shard.
|
||||
|
||||
Build a local validation envelope, start the worker RPC without
|
||||
waiting, join the metadata rendezvous, and require unanimous source,
|
||||
data-plane, and runtime readiness before constructing NCCL.
|
||||
"""
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=cast(M2NWeightSource, source),
|
||||
is_sender=init_info.is_sender,
|
||||
)
|
||||
engine._initialize(init_info, source)
|
||||
return engine
|
||||
|
||||
def _initialize(
|
||||
self,
|
||||
init_info: M2NTrainerInitInfo,
|
||||
source: WeightSource | None,
|
||||
) -> None:
|
||||
"""Run source validation, metadata agreement, and NCCL rendezvous."""
|
||||
self._num_trainer_ranks = init_info.num_trainer_ranks
|
||||
local_source_error: str | None = None
|
||||
try:
|
||||
if not isinstance(source, M2NWeightSource):
|
||||
raise TypeError(
|
||||
"nccl_m2n needs per-parameter layouts, so its source must "
|
||||
f"be an M2NWeightSource; got {type(source).__name__}"
|
||||
)
|
||||
self._m2n = import_m2n()
|
||||
self._prepare_source_plan(source, init_info.num_trainer_ranks)
|
||||
except Exception as exc:
|
||||
local_source_error = f"{type(exc).__name__}: {exc}"
|
||||
self._metas = []
|
||||
self._wire_params = []
|
||||
self._source_digest = source_plan_digest([])
|
||||
|
||||
executor: ThreadPoolExecutor | None = None
|
||||
init_future: Future[None] | None = None
|
||||
if self.is_sender:
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="m2n-weight-sync"
|
||||
)
|
||||
init_future = executor.submit(
|
||||
self.client.init_weight_transfer_engine,
|
||||
self._worker_init_info(init_info),
|
||||
)
|
||||
handle = None
|
||||
group = None
|
||||
try:
|
||||
reservation = init_info.listen_socket
|
||||
try:
|
||||
metadata_group = stateless_init_metadata_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
init_info.rank,
|
||||
init_info.world_size,
|
||||
listen_socket=reservation,
|
||||
)
|
||||
finally:
|
||||
init_info.listen_socket = None
|
||||
if reservation is not None:
|
||||
reservation.close()
|
||||
plan_digest = check_source_plan_agreement(
|
||||
metadata_group,
|
||||
self._wire_params,
|
||||
expected_digest=self._source_digest,
|
||||
local_error=local_source_error,
|
||||
)
|
||||
|
||||
# Retain the common destination mesh and the merged NCCL publication
|
||||
# of per-parameter placements.
|
||||
self._dst_mesh = M2NMesh(
|
||||
init_info.destination_mesh_dims, init_info.num_trainer_ranks
|
||||
)
|
||||
unique_id_bytes: bytes | None = None
|
||||
data_plane_error: str | None = None
|
||||
try:
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
except Exception as exc:
|
||||
data_plane_error = f"{type(exc).__name__}: {exc}"
|
||||
check_data_plane_agreement(
|
||||
metadata_group,
|
||||
unique_id_bytes,
|
||||
data_plane_error,
|
||||
)
|
||||
|
||||
runtime_error: str | None = None
|
||||
nccl = None
|
||||
device = None
|
||||
try:
|
||||
nccl, device, handle = prepare_m2n_local_runtime(
|
||||
self._m2n, init_info.max_cta
|
||||
)
|
||||
except Exception as exc:
|
||||
runtime_error = f"{type(exc).__name__}: {exc}"
|
||||
check_runtime_ready_agreement(metadata_group, runtime_error)
|
||||
assert nccl is not None and device is not None and handle is not None
|
||||
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
|
||||
destination_placements = self._receive_destination_plan(
|
||||
group,
|
||||
init_info.num_trainer_ranks,
|
||||
torch.device("cuda", device),
|
||||
)
|
||||
|
||||
complete_plan_error: str | None = None
|
||||
try:
|
||||
assert self._dst_mesh is not None
|
||||
validate_transfer_plan(
|
||||
self._metas, self._dst_mesh, destination_placements
|
||||
)
|
||||
except Exception as exc:
|
||||
complete_plan_error = f"{type(exc).__name__}: {exc}"
|
||||
assert self._dst_mesh is not None
|
||||
check_complete_plan_agreement(
|
||||
metadata_group,
|
||||
plan_digest,
|
||||
self._dst_mesh,
|
||||
destination_placements,
|
||||
complete_plan_error,
|
||||
)
|
||||
|
||||
if init_future is not None:
|
||||
init_future.result()
|
||||
|
||||
logger.info(
|
||||
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
|
||||
"inference ranks [%d, %d)",
|
||||
len(self._metas),
|
||||
init_info.num_trainer_ranks,
|
||||
init_info.num_trainer_ranks,
|
||||
init_info.world_size,
|
||||
)
|
||||
logger.info("nccl_m2n source plan digest: %s", plan_digest)
|
||||
except BaseException:
|
||||
if group is not None:
|
||||
try:
|
||||
# Transport cleanup only; reload validity belongs to the
|
||||
# model-update lifecycle, not this backend.
|
||||
group.abort()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to abort partially initialized nccl_m2n communicator"
|
||||
)
|
||||
if handle is not None:
|
||||
try:
|
||||
handle.destroy()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to destroy partially initialized nccl_m2n handle"
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
if executor is not None:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
self._metadata_group = metadata_group
|
||||
self._handle = handle
|
||||
self.group = group
|
||||
self._dst_placements = destination_placements
|
||||
|
||||
def _receive_destination_plan(
|
||||
self, group: Any, first_worker_rank: int, device: torch.device
|
||||
) -> list[Placements | None]:
|
||||
"""Receive the inference-side per-parameter placements."""
|
||||
# The first inference worker publishes one placement per parameter.
|
||||
# Every trainer rank receives the complete plan in this single,
|
||||
# initialization-time broadcast.
|
||||
placements = publish_destination_placements(
|
||||
group,
|
||||
device,
|
||||
torch.cuda.current_stream(),
|
||||
first_worker_rank,
|
||||
None,
|
||||
len(self._metas),
|
||||
)
|
||||
return placements
|
||||
|
||||
def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]:
|
||||
"""Handshake payload: rendezvous, both meshes, and the transfer plan."""
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
||||
"master_address": init_info.master_address,
|
||||
"master_port": init_info.master_port,
|
||||
"nccl_unique_id_b64": init_info.nccl_unique_id_b64,
|
||||
"rank_offset": init_info.num_trainer_ranks,
|
||||
"world_size": init_info.world_size,
|
||||
"dst_mesh_dims": list(init_info.destination_mesh_dims),
|
||||
"source_digest": self._source_digest,
|
||||
"params": [param.to_dict() for param in self._wire_params],
|
||||
"max_cta": init_info.max_cta,
|
||||
}
|
||||
return {key: value for key, value in payload.items() if value is not None}
|
||||
|
||||
def _prepare_source_plan(
|
||||
self, source: M2NWeightSource, num_trainer_ranks: int
|
||||
) -> None:
|
||||
"""Build and validate this rank's immutable source plan."""
|
||||
self._metas = source.metadata()
|
||||
names: set[str] = set()
|
||||
for meta in self._metas:
|
||||
if not isinstance(meta, M2NParamMeta):
|
||||
raise TypeError(
|
||||
"nccl_m2n source metadata must contain M2NParamMeta "
|
||||
f"entries, got {type(meta).__name__}"
|
||||
)
|
||||
if meta.name in names:
|
||||
raise ValueError(
|
||||
"nccl_m2n source metadata contains duplicate parameter "
|
||||
f"'{meta.name}'"
|
||||
)
|
||||
names.add(meta.name)
|
||||
check_transferable(meta.name, meta.dtype, meta.shape)
|
||||
layout = meta.source_layout
|
||||
if layout.mesh.start_rank != 0:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source mesh must start at rank 0, "
|
||||
f"got {layout.mesh.start_rank}"
|
||||
)
|
||||
if layout.mesh.size != num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source mesh covers "
|
||||
f"{layout.mesh.size} ranks, but there are "
|
||||
f"{num_trainer_ranks} trainer ranks"
|
||||
)
|
||||
expected_local_shape(meta)
|
||||
self._wire_params = [
|
||||
M2NWireParam(
|
||||
name=meta.name,
|
||||
dtype_name=str(meta.dtype).split(".")[-1],
|
||||
shape=meta.shape,
|
||||
src_mesh_dims=meta.source_layout.mesh.dims,
|
||||
src_placements=meta.source_layout.placements,
|
||||
)
|
||||
for meta in self._metas
|
||||
]
|
||||
self._source_digest = source_plan_digest(self._wire_params)
|
||||
|
||||
def send_weights(self) -> None:
|
||||
"""Drive one update round: start, reshard concurrently, then finish."""
|
||||
if self._handle is None or self.group is None:
|
||||
raise RuntimeError("nccl_m2n trainer engine is not initialized")
|
||||
|
||||
if not self.is_sender:
|
||||
try:
|
||||
self._send()
|
||||
except BaseException:
|
||||
try:
|
||||
self._receive_round_outcome()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to consume nccl_m2n round outcome after local failure"
|
||||
)
|
||||
raise
|
||||
self._receive_round_outcome()
|
||||
return
|
||||
|
||||
try:
|
||||
self.client.start_weight_update()
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="m2n-weight-sync"
|
||||
)
|
||||
try:
|
||||
update = executor.submit(
|
||||
self.client.update_weights,
|
||||
{"names": [m.name for m in self._metas]},
|
||||
)
|
||||
# Surface an immediate worker-side rejection before trainers
|
||||
# enter the collective sequence.
|
||||
if update.done():
|
||||
update.result()
|
||||
self._send()
|
||||
update.result()
|
||||
finally:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
self.client.finish_weight_update()
|
||||
except BaseException as exc:
|
||||
try:
|
||||
self._publish_round_outcome(exc)
|
||||
except BaseException:
|
||||
logger.exception("failed to publish nccl_m2n round failure")
|
||||
raise
|
||||
|
||||
self._publish_round_outcome(None)
|
||||
|
||||
def _publish_round_outcome(self, failure: BaseException | None) -> None:
|
||||
"""Tell non-sender trainer ranks whether worker finalization passed."""
|
||||
if self._num_trainer_ranks <= 1:
|
||||
return
|
||||
if self._metadata_group is None:
|
||||
raise RuntimeError("nccl_m2n metadata group is unavailable")
|
||||
envelope = {
|
||||
"ok": failure is None,
|
||||
"error": (
|
||||
None if failure is None else f"{type(failure).__name__}: {failure}"
|
||||
),
|
||||
}
|
||||
for rank in range(1, self._num_trainer_ranks):
|
||||
self._metadata_group.send_obj(envelope, rank)
|
||||
# Keep rank zero's TCPStore alive until every trainer consumed its
|
||||
# outcome. This also keeps the per-rank message counters aligned.
|
||||
expected_ack = {"phase": "round_outcome_consumed"}
|
||||
for rank in range(1, self._num_trainer_ranks):
|
||||
ack = self._metadata_group.recv_obj(rank)
|
||||
if ack != expected_ack:
|
||||
raise RuntimeError(
|
||||
f"trainer rank {rank} sent an invalid round-outcome acknowledgement"
|
||||
)
|
||||
|
||||
def _receive_round_outcome(self) -> None:
|
||||
"""Wait until trainer rank zero observed every worker's finish RPC."""
|
||||
if self._metadata_group is None:
|
||||
raise RuntimeError("nccl_m2n metadata group is unavailable")
|
||||
outcome = self._metadata_group.recv_obj(0)
|
||||
self._metadata_group.send_obj({"phase": "round_outcome_consumed"}, 0)
|
||||
if not isinstance(outcome, Mapping) or set(outcome) != {"ok", "error"}:
|
||||
raise RuntimeError("trainer rank zero sent an invalid round outcome")
|
||||
ok = outcome["ok"]
|
||||
error = outcome["error"]
|
||||
if not isinstance(ok, bool) or ok != (error is None):
|
||||
raise RuntimeError("trainer rank zero sent an invalid round outcome")
|
||||
if not ok:
|
||||
if not isinstance(error, str) or not error:
|
||||
raise RuntimeError("trainer rank zero sent an invalid round error")
|
||||
raise RuntimeError(
|
||||
"nccl_m2n worker update failed on trainer rank zero: " + error
|
||||
)
|
||||
|
||||
def _send(self) -> None:
|
||||
"""Reshard each local shard into its worker-planned destination."""
|
||||
assert self.group is not None
|
||||
assert self._dst_mesh is not None
|
||||
assert self.source is not None
|
||||
m2n = self._m2n
|
||||
stream = torch.cuda.current_stream()
|
||||
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
# Optimizer work may have used another CUDA stream. Make prior writes
|
||||
# visible before a provider materializes current-stream local shards.
|
||||
torch.accelerator.synchronize(device_index)
|
||||
inflight_buffers: list[Any] = []
|
||||
sent = 0
|
||||
for index, (name, buffer) in enumerate(self.source):
|
||||
if index >= len(self._metas):
|
||||
raise RuntimeError(
|
||||
"weight source yielded more parameters than metadata() "
|
||||
f"declared (first extra parameter: '{name}')"
|
||||
)
|
||||
meta = self._metas[index]
|
||||
if name != meta.name:
|
||||
raise RuntimeError(
|
||||
f"weight source yielded '{name}' at position {index}, but "
|
||||
f"its metadata declared '{meta.name}'; the source must "
|
||||
"iterate in a stable order"
|
||||
)
|
||||
validate_local_buffer(m2n, meta, buffer)
|
||||
src_mesh, src_placements = resolve_layout(
|
||||
meta.source_layout.mesh, meta.source_layout.placements
|
||||
)
|
||||
dst_mesh, dst_placements = resolve_layout(
|
||||
self._dst_mesh, self._dst_placements[index]
|
||||
)
|
||||
raw_pointer_args = {}
|
||||
if isinstance(buffer, int):
|
||||
raw_pointer_args = {
|
||||
"src_local_shape": expected_local_shape(meta),
|
||||
"src_dtype": meta.dtype,
|
||||
}
|
||||
m2n.reshard(
|
||||
buffer,
|
||||
None,
|
||||
self.group,
|
||||
stream,
|
||||
src_mesh=to_mesh(m2n, src_mesh),
|
||||
src_placements=to_placements(m2n, src_placements),
|
||||
dst_mesh=to_mesh(m2n, dst_mesh),
|
||||
dst_placements=to_placements(m2n, dst_placements),
|
||||
handle=self._handle,
|
||||
**raw_pointer_args,
|
||||
)
|
||||
# Let allocator-aware buffers recycle storage in stream order.
|
||||
# Other object owners must stay alive until the final synchronize;
|
||||
# raw-pointer lifetime remains the source's responsibility.
|
||||
record_stream = getattr(buffer, "record_stream", None)
|
||||
if callable(record_stream):
|
||||
record_stream(stream)
|
||||
elif not isinstance(buffer, int):
|
||||
inflight_buffers.append(buffer)
|
||||
sent += 1
|
||||
|
||||
if sent != len(self._metas):
|
||||
missing = self._metas[sent].name
|
||||
raise RuntimeError(
|
||||
f"weight source yielded {sent} parameters, but metadata() "
|
||||
f"declared {len(self._metas)} (first missing: '{missing}')"
|
||||
)
|
||||
|
||||
# As in the other weight-transfer backends, wait before returning
|
||||
# rather than relying on same-stream ordering. The caller may then
|
||||
# safely mutate parameters or continue work on another stream.
|
||||
stream.synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Finish pending GPU work and release M2N communication state."""
|
||||
handle, self._handle = self._handle, None
|
||||
group, self.group = self.group, None
|
||||
self._metadata_group = None
|
||||
self._dst_placements = []
|
||||
self._dst_mesh = None
|
||||
self._m2n = None
|
||||
|
||||
first_error: BaseException | None = None
|
||||
|
||||
def attempt(action: Any, message: str) -> None:
|
||||
nonlocal first_error
|
||||
try:
|
||||
action()
|
||||
except BaseException as exc:
|
||||
if first_error is None:
|
||||
first_error = exc
|
||||
logger.exception(message)
|
||||
|
||||
if handle is not None or group is not None:
|
||||
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
|
||||
if handle is not None:
|
||||
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
|
||||
if group is not None:
|
||||
attempt(group.abort, "failed to abort nccl_m2n communicator")
|
||||
if first_error is not None:
|
||||
raise first_error
|
||||
@@ -1,41 +0,0 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Re-export the weight-transfer backend maintained by ModelExpress.
|
||||
|
||||
Install ModelExpress from the main branch's modelexpress_client/python
|
||||
subdirectory.
|
||||
"""
|
||||
|
||||
try:
|
||||
from modelexpress import configure_vllm_logging
|
||||
from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import (
|
||||
ModelExpressWeightTransferEngine,
|
||||
ModelExpressWeightTransferInitInfo,
|
||||
ModelExpressWeightTransferUpdateInfo,
|
||||
)
|
||||
except ModuleNotFoundError as exc:
|
||||
if exc.name not in {
|
||||
"modelexpress",
|
||||
"modelexpress_rl",
|
||||
"modelexpress_rl.inference",
|
||||
"modelexpress_rl.inference.engines",
|
||||
"modelexpress_rl.inference.engines.vllm",
|
||||
"modelexpress_rl.inference.engines.vllm.weight_transfer_engine",
|
||||
}:
|
||||
raise
|
||||
raise ImportError(
|
||||
"The 'modelexpress' weight transfer backend requires ModelExpress. "
|
||||
"Install it with `uv pip install "
|
||||
"'git+https://github.com/ai-dynamo/modelexpress@main"
|
||||
"#subdirectory=modelexpress_client/python'`."
|
||||
) from exc
|
||||
|
||||
configure_vllm_logging()
|
||||
|
||||
__all__ = [
|
||||
"ModelExpressWeightTransferEngine",
|
||||
"ModelExpressWeightTransferInitInfo",
|
||||
"ModelExpressWeightTransferUpdateInfo",
|
||||
]
|
||||
@@ -1,299 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared NCCL initialization helpers for weight transfer engines.
|
||||
|
||||
The dense (`NCCLWeightTransferEngine`) and sparse
|
||||
(`SparseNCCLWeightTransferEngine`) backends are independent engines that share
|
||||
*only* their process-group initialization. That common logic lives here so the
|
||||
sparse engine does not have to subclass the dense one.
|
||||
"""
|
||||
|
||||
import socket
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCL_UNIQUE_ID_BYTES,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
)
|
||||
from vllm.utils.nccl import unpinned_nccl_env
|
||||
|
||||
|
||||
def decode_nccl_unique_id(
|
||||
*,
|
||||
master_address: str | None,
|
||||
master_port: int | None,
|
||||
nccl_unique_id_b64: str | None,
|
||||
ctx: str,
|
||||
) -> bytes | None:
|
||||
"""Validate the rendezvous mode and decode a pre-shared unique id."""
|
||||
has_uid = bool(nccl_unique_id_b64)
|
||||
has_addr = master_address is not None
|
||||
has_port = master_port is not None
|
||||
|
||||
if has_uid and (has_addr or has_port):
|
||||
raise ValueError(
|
||||
f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both"
|
||||
)
|
||||
if not has_uid:
|
||||
if not (has_addr and has_port):
|
||||
raise ValueError(
|
||||
f"{ctx}: need nccl_unique_id_b64, or both master_address and "
|
||||
f"master_port (got master_address={master_address!r}, "
|
||||
f"master_port={master_port!r})"
|
||||
)
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(nccl_unique_id_b64, validate=True)
|
||||
except (ValueError, TypeError) as e:
|
||||
raise ValueError(
|
||||
f"{ctx}: nccl_unique_id_b64 is not valid standard base64 "
|
||||
f"(URL-safe base64 is not accepted)"
|
||||
) from e
|
||||
if len(decoded) != NCCL_UNIQUE_ID_BYTES:
|
||||
raise ValueError(
|
||||
f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} "
|
||||
f"bytes, got {len(decoded)}"
|
||||
)
|
||||
return decoded
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side initialization info for NCCL-based weight transfer backends.
|
||||
|
||||
Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field
|
||||
means the rendezvous fields can no longer keep a fixed positional slot, so a
|
||||
stale positional call fails loudly instead of silently swapping arguments.
|
||||
|
||||
Provide exactly one rendezvous mode:
|
||||
|
||||
* ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup``
|
||||
rendezvous (requires torch on every rank, including the trainer), or
|
||||
* ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the
|
||||
128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX)
|
||||
that mint the unique id out of band and share it (over HTTP, etc.). Note a
|
||||
JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``.
|
||||
|
||||
On the unique-id path all ranks must enter init concurrently (there is no
|
||||
store barrier), and every peer must honor the warm-up handshake: the
|
||||
worker's communicator issues a one-element ``all_reduce`` immediately after
|
||||
``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a
|
||||
foreign peer must issue a matching one-element ``all_reduce`` before any
|
||||
other collective or all ranks deadlock.
|
||||
"""
|
||||
|
||||
master_address: str | None = None
|
||||
master_port: int | None = None
|
||||
rank_offset: int
|
||||
world_size: int
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
packed: bool = False
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_ = self.nccl_unique_id_bytes
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
return decode_nccl_unique_id(
|
||||
master_address=self.master_address,
|
||||
master_port=self.master_port,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="NCCLWeightTransferInitInfo",
|
||||
)
|
||||
|
||||
|
||||
def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
|
||||
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
|
||||
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
|
||||
only the mode actually in use. Shared by the dense and sparse trainer
|
||||
engines so the two cannot drift."""
|
||||
return {key: value for key, value in asdict(init_info).items() if value is not None}
|
||||
|
||||
|
||||
class NCCLRendezvous(Protocol):
|
||||
"""The TCP rendezvous fields `trainer_init` needs.
|
||||
|
||||
Structural so each backend can keep its own trainer init info next to its
|
||||
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
|
||||
this shared module importing either.
|
||||
"""
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
|
||||
|
||||
def stateless_init_metadata_group(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
*,
|
||||
listen_socket: socket.socket | None = None,
|
||||
) -> "StatelessProcessGroup":
|
||||
"""Create only the TCPStore group used for metadata agreement."""
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
return StatelessProcessGroup.create(
|
||||
host=master_address,
|
||||
port=master_port,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
listen_socket=listen_socket,
|
||||
)
|
||||
|
||||
|
||||
def stateless_init_process_group(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""VLLM provides `StatelessProcessGroup` to create a process group
|
||||
without considering the global process group in torch.distributed.
|
||||
It is recommended to create `StatelessProcessGroup`, and then initialize
|
||||
the data-plane communication (NCCL) between external (train processes)
|
||||
and vLLM workers.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
pg = StatelessProcessGroup.create(
|
||||
host=master_address, port=master_port, rank=rank, world_size=world_size
|
||||
)
|
||||
with unpinned_nccl_env():
|
||||
return PyNcclCommunicator(pg, device=device)
|
||||
|
||||
|
||||
def uid_init_process_group(
|
||||
nccl_unique_id_bytes: bytes,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Join the NCCL group from pre-shared ``ncclUniqueId`` bytes.
|
||||
|
||||
The torch-free rendezvous alternative to `stateless_init_process_group`: no
|
||||
TCPStore, and therefore no barrier -- every rank must enter concurrently.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
with unpinned_nccl_env():
|
||||
return PyNcclCommunicator.from_unique_id_bytes(
|
||||
nccl_unique_id_bytes,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
def _worker_group_rank(rank_offset: int, parallel_config: "ParallelConfig") -> int:
|
||||
"""Return this worker's rank in a trainer-to-worker process group."""
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
world_size_per_dp = parallel_config.world_size # TP * PP
|
||||
rank_within_dp = parallel_config.rank
|
||||
return rank_offset + dp_rank * world_size_per_dp + rank_within_dp
|
||||
|
||||
|
||||
def worker_init_metadata_group(
|
||||
init_info: NCCLWeightTransferInitInfo,
|
||||
parallel_config: "ParallelConfig",
|
||||
) -> "StatelessProcessGroup":
|
||||
"""Join the trainer-to-worker TCP metadata group without constructing NCCL."""
|
||||
assert init_info.master_address is not None
|
||||
assert init_info.master_port is not None
|
||||
return stateless_init_metadata_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
_worker_group_rank(init_info.rank_offset, parallel_config),
|
||||
init_info.world_size,
|
||||
)
|
||||
|
||||
|
||||
def worker_init_process_group(
|
||||
init_info: NCCLWeightTransferInitInfo,
|
||||
parallel_config: "ParallelConfig",
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Create the trainer<->worker NCCL group on an inference worker.
|
||||
|
||||
Computes a unique rank for this worker across all data-parallel groups and
|
||||
joins the trainer via whichever rendezvous mode `init_info` carries.
|
||||
"""
|
||||
rank = _worker_group_rank(init_info.rank_offset, parallel_config)
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
if unique_id_bytes is not None:
|
||||
return uid_init_process_group(
|
||||
unique_id_bytes,
|
||||
rank,
|
||||
init_info.world_size,
|
||||
device,
|
||||
)
|
||||
# __post_init__ guarantees the TCP pair when there is no unique id.
|
||||
assert init_info.master_address is not None
|
||||
assert init_info.master_port is not None
|
||||
return stateless_init_process_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
rank,
|
||||
init_info.world_size,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
def trainer_init(
|
||||
init_info: NCCLRendezvous | dict,
|
||||
rank: int = 0,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Initialize NCCL process group for trainer-side weight transfer.
|
||||
|
||||
Uses the current CUDA device (torch.accelerator.current_device_index()).
|
||||
|
||||
Args:
|
||||
init_info: Any object carrying the `NCCLRendezvous` fields (a trainer or
|
||||
worker NCCL init info), or a dict with keys:
|
||||
- master_address: str
|
||||
- master_port: int
|
||||
- world_size: int
|
||||
rank: This trainer process's rank in the group. The broadcast backends
|
||||
have a single trainer at rank 0; multi-rank trainers (m2n) occupy
|
||||
ranks `[0, num_trainer_ranks)` and the workers start after them.
|
||||
|
||||
Returns:
|
||||
PyNcclCommunicator for weight transfer.
|
||||
|
||||
"""
|
||||
if isinstance(init_info, dict):
|
||||
master_address = init_info["master_address"]
|
||||
master_port = init_info["master_port"]
|
||||
world_size = init_info["world_size"]
|
||||
else:
|
||||
master_address = init_info.master_address
|
||||
master_port = init_info.master_port
|
||||
world_size = init_info.world_size
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
return stateless_init_process_group(
|
||||
master_address,
|
||||
master_port,
|
||||
rank,
|
||||
world_size,
|
||||
device,
|
||||
)
|
||||
@@ -1,461 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NCCL-based (dense) weight transfer engine."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar, Self
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
ParamMeta,
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngine,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
worker_init_payload,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
trainer_init as open_trainer_endpoint,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
packed_nccl_broadcast_consumer,
|
||||
packed_nccl_broadcast_producer,
|
||||
)
|
||||
|
||||
# NCCLWeightTransferInitInfo is re-exported here for convenience; its canonical
|
||||
# home is nccl_common, shared with the sparse backend.
|
||||
__all__ = [
|
||||
"NCCLWeightTransferInitInfo",
|
||||
"NCCLTrainerInitInfo",
|
||||
"NCCLWeightTransferUpdateInfo",
|
||||
"NCCLWeightTransferEngine",
|
||||
"NCCLTrainerWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for the dense NCCL weight transfer backend.
|
||||
|
||||
The sender opens its endpoint as NCCL rank 0, so it needs no `rank_offset`.
|
||||
`world_size` is the full trainer+worker NCCL group size. `rank` (from
|
||||
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
|
||||
|
||||
The trainer joins over a TCPStore rendezvous (`master_address` +
|
||||
`master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore
|
||||
mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a
|
||||
`nccl_unique_id_b64` payload straight to the inference workers'
|
||||
`init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than
|
||||
going through this engine.
|
||||
|
||||
`packed` / buffer sizes are the transfer's wire params. The trainer
|
||||
propagates them to the worker at `trainer_init` so the two sides cannot
|
||||
disagree. Note this defaults to packed, unlike the worker-side
|
||||
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
|
||||
ships a value. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "nccl"
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
packed: bool = True
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Per-round update info for the dense NCCL weight transfer backend.
|
||||
|
||||
Whether the transfer is packed (and the buffer geometry) is a must-agree
|
||||
wire param carried on the init info (`NCCLTrainerInitInfo` /
|
||||
`NCCLWeightTransferInitInfo`), not here; this carries only the per-round
|
||||
parameter metadata.
|
||||
"""
|
||||
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate that all lists have the same length."""
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
f"`dtype_names` should be of the same size as `names`: "
|
||||
f"got {len(self.dtype_names)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.shapes) != num_params:
|
||||
raise ValueError(
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
|
||||
|
||||
class NCCLWeightTransferEngine(
|
||||
WeightTransferEngine[NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo]
|
||||
):
|
||||
"""Weight transfer engine using NCCL for communication between trainer and workers.
|
||||
|
||||
This implementation uses NCCL broadcast operations to transfer dense
|
||||
checkpoint-format weights from the trainer (rank 0) to all inference workers
|
||||
in a process group. Received weights are loaded via the model's
|
||||
`load_weights` using the layerwise reload lifecycle.
|
||||
"""
|
||||
|
||||
# Define backend-specific dataclass types
|
||||
init_info_cls = NCCLWeightTransferInitInfo
|
||||
update_info_cls = NCCLWeightTransferUpdateInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
# Set from the trainer-supplied init info at the handshake; defaults are
|
||||
# only for the (unreachable) receive-before-init case.
|
||||
self.packed = False
|
||||
self.packed_buffer_size_bytes = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
"""Initialize NCCL process group with the trainer and record the
|
||||
trainer-supplied wire params so the worker decodes exactly as the
|
||||
trainer encodes.
|
||||
|
||||
Args:
|
||||
init_info: NCCL initialization info containing master address, port,
|
||||
rank offset, world size, and the packed wire params
|
||||
|
||||
"""
|
||||
self.packed = init_info.packed
|
||||
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
|
||||
self.packed_num_buffers = init_info.packed_num_buffers
|
||||
self.model_update_group = worker_init_process_group(
|
||||
init_info, self.parallel_config
|
||||
)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Initialize layerwise reloading for the incoming checkpoint weights."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading after all weights have been received."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
|
||||
"""Receive weights from trainer via NCCL broadcast.
|
||||
|
||||
Whether to use packed broadcasting (and the buffer geometry) is read
|
||||
from `self.packed` / `self.packed_*`, set at the init handshake from the
|
||||
trainer's init info, so it is guaranteed to match how the trainer
|
||||
encoded.
|
||||
|
||||
Args:
|
||||
update_info: NCCL update info containing parameter names, dtypes,
|
||||
and shapes
|
||||
|
||||
"""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
disable_mtp_completeness_check,
|
||||
)
|
||||
|
||||
with disable_mtp_completeness_check():
|
||||
if self.packed:
|
||||
# Build iterator of (name, (shape, dtype)) from update_info
|
||||
def state_dict_info_iterator():
|
||||
for name, dtype_name, shape in zip(
|
||||
update_info.names, update_info.dtype_names, update_info.shapes
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
yield (name, (shape, dtype))
|
||||
|
||||
packed_nccl_broadcast_consumer(
|
||||
iterator=state_dict_info_iterator(),
|
||||
group=self.model_update_group,
|
||||
src=0,
|
||||
post_unpack_func=self.model.load_weights,
|
||||
buffer_size_bytes=self.packed_buffer_size_bytes,
|
||||
num_buffers=self.packed_num_buffers,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
# Use simple one-by-one broadcasting
|
||||
for name, dtype_name, shape in zip(
|
||||
update_info.names, update_info.dtype_names, update_info.shapes
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
weight = torch.empty(shape, dtype=dtype, device=self.device)
|
||||
self.model_update_group.broadcast(
|
||||
weight, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self.model.load_weights([(name, weight)])
|
||||
del weight
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
|
||||
|
||||
class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerInitInfo]):
|
||||
"""Trainer-side NCCL weight transfer engine.
|
||||
|
||||
On the sender (rank 0) holds the NCCL communicator and drives the full
|
||||
update round trip: it runs the inference-side `update_weights` concurrently
|
||||
with the trainer-side broadcast (both rendezvous inside the same NCCL
|
||||
calls), then finishes the update. Non-sender trainer ranks hold no
|
||||
communicator; they only iterate the source to stay in the trainer-side
|
||||
collective (e.g. FSDP `full_tensor()`) and skip the client RPCs and the
|
||||
broadcast (all guarded on `is_sender`).
|
||||
|
||||
`packed` / buffer sizes come from `NCCLTrainerInitInfo`; the sender
|
||||
propagates them to the worker at `trainer_init` (on the worker-side init
|
||||
info), so per-round payloads carry only parameter metadata.
|
||||
"""
|
||||
|
||||
init_info_cls = NCCLTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource,
|
||||
is_sender: bool = True,
|
||||
packed: bool = True,
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
) -> None:
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.packed = packed
|
||||
self.packed_buffer_size_bytes = packed_buffer_size_bytes
|
||||
self.packed_num_buffers = packed_num_buffers
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: NCCLTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
if source is None:
|
||||
raise ValueError("NCCL trainer weight transfer requires a WeightSource.")
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=source,
|
||||
is_sender=init_info.is_sender,
|
||||
packed=init_info.packed,
|
||||
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
|
||||
packed_num_buffers=init_info.packed_num_buffers,
|
||||
)
|
||||
if not engine.is_sender:
|
||||
# Non-sender trainer ranks aren't part of the transfer NCCL group and
|
||||
# don't drive the inference side; they only participate in the
|
||||
# trainer-side gather during send_weights.
|
||||
return engine
|
||||
|
||||
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
|
||||
worker_init_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=1,
|
||||
world_size=init_info.world_size,
|
||||
packed=init_info.packed,
|
||||
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
|
||||
packed_num_buffers=init_info.packed_num_buffers,
|
||||
)
|
||||
|
||||
# The inference workers block inside init_weight_transfer_engine waiting
|
||||
# for the NCCL rendezvous, so we kick that off on a side thread while we
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine,
|
||||
worker_init_payload(worker_init_info),
|
||||
)
|
||||
engine.model_update_group = open_trainer_endpoint(init_info)
|
||||
future.result() # surface any inference-side init error
|
||||
|
||||
return engine
|
||||
|
||||
def send_weights(self) -> None:
|
||||
assert self.source is not None # guaranteed by trainer_init / __init__
|
||||
source = self.source
|
||||
|
||||
# Metadata is declared without gathering. For Megatron it is itself a
|
||||
# collective, so every rank runs it; only the sender ships it.
|
||||
meta = source.metadata()
|
||||
|
||||
if not self.is_sender:
|
||||
# Non-sender ranks only join the trainer-side gather collective.
|
||||
self._broadcast(source, meta)
|
||||
self._post_send_sync()
|
||||
return
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=[m.name for m in meta],
|
||||
dtype_names=[str(m.dtype).split(".")[-1] for m in meta],
|
||||
shapes=[list(m.shape) for m in meta],
|
||||
)
|
||||
|
||||
self.client.start_weight_update()
|
||||
# update_weights (workers receive) must run concurrently with the
|
||||
# trainer-side broadcast — both rendezvous inside the same NCCL calls.
|
||||
exe = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
future = exe.submit(self.client.update_weights, asdict(update_info))
|
||||
# Cheap best-effort: if update_weights already failed (e.g. a bad
|
||||
# request rejected before any NCCL call), surface it now instead of
|
||||
# hanging in broadcast waiting for a peer that will never arrive.
|
||||
if future.done():
|
||||
future.result()
|
||||
self._broadcast(source, meta)
|
||||
future.result() # surface inference-side errors
|
||||
finally:
|
||||
# Never wait for the RPC thread here. If the broadcast raised, the
|
||||
# worker is still blocked in the matching NCCL call and will never
|
||||
# return, so joining would turn the error into a permanent hang.
|
||||
# Let the exception out instead; the transfer group is unusable
|
||||
# either way and the caller has to tear it down. (The orphaned
|
||||
# thread is still joined at interpreter exit, so a caller that wants
|
||||
# to exit cleanly after such a failure must not wait on it.)
|
||||
exe.shutdown(wait=False)
|
||||
self.client.finish_weight_update()
|
||||
self._post_send_sync()
|
||||
|
||||
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
|
||||
"""Iterate the source (materializing each tensor — a collective on all
|
||||
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
|
||||
Non-sender ranks only replay the iteration to stay in the collective."""
|
||||
if not self.is_sender:
|
||||
for _ in source:
|
||||
pass
|
||||
return
|
||||
|
||||
assert self.model_update_group is not None, (
|
||||
"trainer_init() must be called before _broadcast()."
|
||||
)
|
||||
pairs = self._checked_iter(source, meta)
|
||||
if self.packed:
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=pairs,
|
||||
group=self.model_update_group,
|
||||
src=0,
|
||||
post_iter_func=lambda item: item[1],
|
||||
buffer_size_bytes=self.packed_buffer_size_bytes,
|
||||
num_buffers=self.packed_num_buffers,
|
||||
)
|
||||
else:
|
||||
stream = torch.cuda.current_stream()
|
||||
for _name, tensor in pairs:
|
||||
# NCCL sends `numel` elements straight from `data_ptr()`, so a
|
||||
# non-contiguous view would ship whatever follows its base
|
||||
# pointer. Keep the copy referenced until the broadcast is
|
||||
# enqueued. (The packed path linearizes in `pack_tensors`.)
|
||||
send = tensor if tensor.is_contiguous() else tensor.contiguous()
|
||||
self.model_update_group.broadcast(send, src=0, stream=stream)
|
||||
|
||||
@staticmethod
|
||||
def _checked_iter(
|
||||
source: WeightSource, meta: list[ParamMeta]
|
||||
) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Yield the source's pairs, checking each against what the worker was
|
||||
told to expect.
|
||||
|
||||
The worker sizes its receive buffers — and in packed mode cuts its chunk
|
||||
boundaries — from the update info, which is built from `metadata()`. If
|
||||
iteration disagrees with it, the two sides split the stream differently
|
||||
and the transfer hangs in NCCL or loads garbage. Checking here costs one
|
||||
comparison per parameter and turns that into an error naming the first
|
||||
divergent parameter. Sender-only: under pipeline parallelism a
|
||||
non-sender's yielded tensor is not meaningful.
|
||||
"""
|
||||
sent = 0
|
||||
for name, tensor in source:
|
||||
if sent >= len(meta):
|
||||
raise ValueError(
|
||||
f"WeightSource yielded more parameters than metadata() "
|
||||
f"declared ({len(meta)}); first extra is {name!r}."
|
||||
)
|
||||
expected = meta[sent]
|
||||
if (
|
||||
name != expected.name
|
||||
or tensor.dtype != expected.dtype
|
||||
or tuple(tensor.shape) != expected.shape
|
||||
):
|
||||
raise ValueError(
|
||||
"WeightSource metadata() disagrees with iteration at index "
|
||||
f"{sent}: declared {expected.name!r} "
|
||||
f"{expected.dtype} {tuple(expected.shape)}, got {name!r} "
|
||||
f"{tensor.dtype} {tuple(tensor.shape)}. Both channels must "
|
||||
"enumerate the same parameters in the same order."
|
||||
)
|
||||
sent += 1
|
||||
yield name, tensor
|
||||
if sent != len(meta):
|
||||
raise ValueError(
|
||||
f"WeightSource yielded {sent} parameters but metadata() "
|
||||
f"declared {len(meta)}; the worker is waiting for the rest."
|
||||
)
|
||||
|
||||
def _post_send_sync(self) -> None:
|
||||
"""Wait for this rank's transfer work to land before returning.
|
||||
|
||||
Broadcasts are only *enqueued* by `send_weights`: the unpacked path on
|
||||
the current stream, the packed path on the producer's own streams (which
|
||||
it drains itself). Waiting here lets a caller mutate parameters, or
|
||||
start the next step on another stream, as soon as `send_weights`
|
||||
returns, instead of silently depending on same-stream ordering. Every
|
||||
rank waits: a non-sender's `full_tensor()` gathers feed the sender's
|
||||
broadcast, so they must have landed before it may touch its shards.
|
||||
|
||||
Unlike IPC there is no cross-rank barrier here. Nothing in this backend
|
||||
outlives the collective it travelled in (IPC's barrier keeps shared
|
||||
buffers alive until every consumer has opened them), so a barrier would
|
||||
only add a dependency on the default process group that this backend
|
||||
otherwise does not have.
|
||||
"""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
@@ -1,494 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Packed tensor utilities for efficient weight transfer."""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.multiprocessing.reductions import reduce_tensor
|
||||
|
||||
# Default values for packed tensor transfer.
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024 # 1GB
|
||||
DEFAULT_PACKED_NUM_BUFFERS = 2
|
||||
|
||||
|
||||
# Streams are cached across calls: the caching allocator keys cached blocks by the
|
||||
# stream they were allocated on, so fresh streams at every call would strand the
|
||||
# packed buffers' reserved memory per call. See gh-52950.
|
||||
@cache
|
||||
def _get_streams(device_idx: int, num_buffers: int) -> tuple[torch.cuda.Stream, ...]:
|
||||
return tuple(torch.cuda.Stream(device=device_idx) for _ in range(num_buffers))
|
||||
|
||||
|
||||
def unpack_tensor(
|
||||
packed_tensor: torch.Tensor,
|
||||
names: list[str],
|
||||
shapes: list[list[int]],
|
||||
dtypes: list[torch.dtype],
|
||||
tensor_sizes: list[int],
|
||||
) -> list[tuple[str, torch.Tensor]]:
|
||||
"""Unpack a packed uint8 tensor into a list of named tensors.
|
||||
|
||||
The returned tensors are **views** of ``packed_tensor`` (the
|
||||
``.contiguous()`` call is a no-op on already-contiguous row-slices).
|
||||
If ``packed_tensor`` lives in storage that may be reused — e.g. a
|
||||
reused CUDA IPC buffer — callers must clone the results before the
|
||||
underlying storage is overwritten.
|
||||
|
||||
Args:
|
||||
packed_tensor: The packed torch.uint8 tensor to unpack
|
||||
names: List of tensor names
|
||||
shapes: List of tensor shapes
|
||||
dtypes: List of tensor dtypes
|
||||
tensor_sizes: List of tensor sizes in bytes
|
||||
|
||||
"""
|
||||
unpacked_tensors = packed_tensor.split(tensor_sizes)
|
||||
|
||||
return [
|
||||
(name, tensor.contiguous().view(dtype).view(shape))
|
||||
for name, shape, dtype, tensor in zip(names, shapes, dtypes, unpacked_tensors)
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackedChunk:
|
||||
"""Result of packing tensors into a single contiguous uint8 buffer."""
|
||||
|
||||
packed_tensor: torch.Tensor
|
||||
names: list[str]
|
||||
shapes: list[list[int]]
|
||||
dtypes: list[torch.dtype]
|
||||
tensor_sizes: list[int]
|
||||
|
||||
|
||||
def pack_tensors(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
|
||||
buffer_size_bytes: int,
|
||||
tensor_list: list[torch.Tensor] | None = None,
|
||||
current_size: int = 0,
|
||||
) -> PackedChunk | None:
|
||||
"""Pack tensors from an iterator into a single contiguous uint8 buffer.
|
||||
|
||||
Consumes from the iterator until the accumulated size exceeds
|
||||
buffer_size_bytes or the iterator is exhausted, then returns a
|
||||
PackedChunk. Returns None if no tensors were consumed.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of (name, tensor) pairs
|
||||
post_iter_func: Applied to each item before linearizing to uint8
|
||||
buffer_size_bytes: Max bytes before flushing
|
||||
tensor_list: Pre-existing tensor list to append to (for NCCL
|
||||
multi-buffer reuse). If None, a fresh list is created.
|
||||
current_size: Byte count already accumulated in tensor_list
|
||||
|
||||
"""
|
||||
if tensor_list is None:
|
||||
tensor_list = []
|
||||
|
||||
names: list[str] = []
|
||||
shapes: list[list[int]] = []
|
||||
dtypes: list[torch.dtype] = []
|
||||
tensor_sizes: list[int] = []
|
||||
total_bytes = current_size
|
||||
|
||||
while True:
|
||||
try:
|
||||
item = next(iterator)
|
||||
except StopIteration:
|
||||
break
|
||||
|
||||
name, orig_tensor = item
|
||||
# Apply post processing and convert to linearized uint8 tensor
|
||||
tensor = post_iter_func(item).contiguous().view(-1).view(torch.uint8)
|
||||
|
||||
if tensor.numel() > buffer_size_bytes:
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
f"Tensor '{name}' has size {tensor.numel()} bytes, which "
|
||||
f"exceeds buffer_size_bytes={buffer_size_bytes}.",
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
tensor_list.append(tensor)
|
||||
names.append(name)
|
||||
shapes.append(list(orig_tensor.shape))
|
||||
dtypes.append(orig_tensor.dtype)
|
||||
tensor_sizes.append(tensor.numel())
|
||||
total_bytes += tensor.numel()
|
||||
|
||||
if total_bytes > buffer_size_bytes:
|
||||
break
|
||||
|
||||
if not tensor_list:
|
||||
return None
|
||||
|
||||
packed = torch.cat(tensor_list, dim=0)
|
||||
del tensor_list
|
||||
return PackedChunk(
|
||||
packed_tensor=packed,
|
||||
names=names,
|
||||
shapes=shapes,
|
||||
dtypes=dtypes,
|
||||
tensor_sizes=tensor_sizes,
|
||||
)
|
||||
|
||||
|
||||
# ── NCCL packed broadcast ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def packed_nccl_broadcast_producer(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
group: Any,
|
||||
src: int,
|
||||
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
|
||||
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
) -> None:
|
||||
"""Broadcast tensors in a packed manner from trainer to workers.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of model parameters. Returns a tuple of (name, tensor)
|
||||
group: Process group (PyNcclCommunicator)
|
||||
src: Source rank (0 in current implementation)
|
||||
post_iter_func: Function to apply to each (name, tensor) pair before
|
||||
packing, should return a tensor
|
||||
buffer_size_bytes: Size in bytes for each packed tensor buffer.
|
||||
Both producer and consumer must use the same value.
|
||||
num_buffers: Number of buffers for double/triple buffering.
|
||||
Both producer and consumer must use the same value.
|
||||
|
||||
"""
|
||||
streams = _get_streams(torch.accelerator.current_device_index(), num_buffers)
|
||||
# Keep references to in-flight chunks so their packed_tensors
|
||||
# aren't freed while an async broadcast is still reading them.
|
||||
in_flight: list[PackedChunk | None] = [None] * num_buffers
|
||||
buffer_idx = 0
|
||||
|
||||
while True:
|
||||
# Synchronize the current stream
|
||||
streams[buffer_idx].synchronize()
|
||||
# Previous chunk on this buffer slot is now safe to free
|
||||
in_flight[buffer_idx] = None
|
||||
# Start tasks for the new buffer in a new stream
|
||||
with torch.cuda.stream(streams[buffer_idx]):
|
||||
chunk = pack_tensors(iterator, post_iter_func, buffer_size_bytes)
|
||||
if chunk is None:
|
||||
break
|
||||
# Pack the tensors and call broadcast collective
|
||||
group.broadcast(chunk.packed_tensor, src=src)
|
||||
# Hold reference until this stream is synchronized
|
||||
in_flight[buffer_idx] = chunk
|
||||
# Move to the next buffer
|
||||
buffer_idx = (buffer_idx + 1) % num_buffers
|
||||
|
||||
# Drain every slot before returning.
|
||||
for stream in streams:
|
||||
stream.synchronize()
|
||||
|
||||
|
||||
def packed_nccl_broadcast_consumer(
|
||||
iterator: Iterator[tuple[str, tuple[list[int], torch.dtype]]],
|
||||
group: Any,
|
||||
src: int,
|
||||
post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
device: torch.device | str = "cuda",
|
||||
) -> None:
|
||||
"""Consume packed tensors and unpack them into a list of tensors.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of parameter metadata. Returns (name, (shape, dtype))
|
||||
group: Process group (PyNcclCommunicator)
|
||||
src: Source rank (0 in current implementation)
|
||||
post_unpack_func: Function to apply to each list of (name, tensor) after
|
||||
unpacking
|
||||
buffer_size_bytes: Size in bytes for each packed tensor buffer.
|
||||
Both producer and consumer must use the same value.
|
||||
num_buffers: Number of buffers for double/triple buffering.
|
||||
Both producer and consumer must use the same value.
|
||||
device: Device for the receive buffers. Must match the device the NCCL
|
||||
communicator was created on (the worker's assigned device).
|
||||
|
||||
"""
|
||||
target_packed_tensor_size = buffer_size_bytes
|
||||
|
||||
streams = _get_streams(torch.accelerator.current_device_index(), num_buffers)
|
||||
buffer_idx = 0
|
||||
|
||||
packing_tensor_meta_data: list[list[tuple[str, list[int], torch.dtype, int]]] = [
|
||||
[] for _ in range(num_buffers)
|
||||
]
|
||||
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
|
||||
packed_tensors: list[torch.Tensor] = [
|
||||
torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers)
|
||||
]
|
||||
|
||||
while True:
|
||||
# Synchronize the current stream
|
||||
streams[buffer_idx].synchronize()
|
||||
with torch.cuda.stream(streams[buffer_idx]):
|
||||
# Initialize the packing tensor meta data
|
||||
packing_tensor_meta_data[buffer_idx] = []
|
||||
packing_tensor_sizes[buffer_idx] = 0
|
||||
try:
|
||||
# Form a packed tensor
|
||||
while True:
|
||||
name, (shape, dtype) = next(iterator)
|
||||
tensor_size = math.prod(shape) * dtype.itemsize
|
||||
packing_tensor_meta_data[buffer_idx].append(
|
||||
(name, shape, dtype, tensor_size)
|
||||
)
|
||||
packing_tensor_sizes[buffer_idx] += tensor_size
|
||||
if packing_tensor_sizes[buffer_idx] > target_packed_tensor_size:
|
||||
break
|
||||
# Create a packed tensor and broadcast it
|
||||
packed_tensors[buffer_idx] = torch.empty(
|
||||
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device
|
||||
)
|
||||
group.broadcast(packed_tensors[buffer_idx], src=src)
|
||||
# Load the packed tensor into the model
|
||||
names, shapes, dtypes, tensor_sizes = zip(
|
||||
*packing_tensor_meta_data[buffer_idx]
|
||||
)
|
||||
post_unpack_func(
|
||||
unpack_tensor(
|
||||
packed_tensors[buffer_idx],
|
||||
list(names),
|
||||
list(shapes),
|
||||
list(dtypes),
|
||||
list(tensor_sizes),
|
||||
)
|
||||
)
|
||||
# Move to the next buffer
|
||||
buffer_idx = (buffer_idx + 1) % num_buffers
|
||||
except StopIteration:
|
||||
# Do the last broadcast if there are remaining tensors
|
||||
if len(packing_tensor_meta_data[buffer_idx]) > 0:
|
||||
# Create a packed tensor and broadcast it
|
||||
packed_tensors[buffer_idx] = torch.empty(
|
||||
packing_tensor_sizes[buffer_idx],
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
group.broadcast(packed_tensors[buffer_idx], src=src)
|
||||
# Load the packed tensor into the model
|
||||
names, shapes, dtypes, tensor_sizes = zip(
|
||||
*packing_tensor_meta_data[buffer_idx]
|
||||
)
|
||||
post_unpack_func(
|
||||
unpack_tensor(
|
||||
packed_tensors[buffer_idx],
|
||||
list(names),
|
||||
list(shapes),
|
||||
list(dtypes),
|
||||
list(tensor_sizes),
|
||||
)
|
||||
)
|
||||
break
|
||||
|
||||
# Drain every slot before returning.
|
||||
for stream in streams:
|
||||
stream.synchronize()
|
||||
|
||||
|
||||
# ── IPC packed transfer ────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackedIpcChunk:
|
||||
"""Metadata and IPC handle for a single packed chunk."""
|
||||
|
||||
names: list[str]
|
||||
shapes: list[list[int]]
|
||||
dtype_names: list[str]
|
||||
tensor_sizes: list[int]
|
||||
ipc_handle: dict[str, tuple]
|
||||
|
||||
|
||||
def packed_ipc_producer(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
gpu_uuid: str,
|
||||
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
|
||||
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
) -> Iterator[PackedIpcChunk]:
|
||||
"""Pack tensors into a reusable IPC buffer and yield handles.
|
||||
|
||||
Allocates a single GPU buffer of ``buffer_size_bytes`` and registers
|
||||
it for IPC once via ``reduce_tensor``. Each chunk's packed data is
|
||||
copied into this buffer before yielding, so only one IPC-shared
|
||||
allocation exists for the lifetime of the transfer.
|
||||
|
||||
Callers **must** ensure the consumer has finished reading the buffer
|
||||
(e.g. ``ray.get`` returned) before resuming the generator for the
|
||||
next chunk.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of (name, tensor) pairs.
|
||||
gpu_uuid: Physical GPU UUID string for this rank.
|
||||
post_iter_func: Applied to each (name, tensor) before packing.
|
||||
buffer_size_bytes: Exact capacity of the reusable IPC buffer.
|
||||
Every chunk is guaranteed to fit within this size. A
|
||||
``ValueError`` is raised if any single tensor exceeds it.
|
||||
|
||||
"""
|
||||
ipc_buffer = torch.empty(buffer_size_bytes, dtype=torch.uint8, device="cuda")
|
||||
_, ipc_args = reduce_tensor(ipc_buffer)
|
||||
|
||||
names: list[str] = []
|
||||
shapes: list[list[int]] = []
|
||||
dtypes: list[torch.dtype] = []
|
||||
tensor_sizes: list[int] = []
|
||||
total_bytes = 0
|
||||
|
||||
for name, orig_tensor in iterator:
|
||||
flat = (
|
||||
post_iter_func((name, orig_tensor)).contiguous().view(-1).view(torch.uint8)
|
||||
)
|
||||
|
||||
if flat.numel() > buffer_size_bytes:
|
||||
raise ValueError(
|
||||
f"Tensor '{name}' has size {flat.numel()} bytes, "
|
||||
f"which exceeds buffer_size_bytes={buffer_size_bytes}. "
|
||||
f"Increase buffer_size_bytes to at least {flat.numel()}."
|
||||
)
|
||||
|
||||
if total_bytes and total_bytes + flat.numel() > buffer_size_bytes:
|
||||
# Drain queued copies so the consumer sees a fully-written buffer.
|
||||
torch.cuda.current_stream().synchronize()
|
||||
yield PackedIpcChunk(
|
||||
names=names,
|
||||
shapes=shapes,
|
||||
dtype_names=[str(d).split(".")[-1] for d in dtypes],
|
||||
tensor_sizes=tensor_sizes,
|
||||
ipc_handle={gpu_uuid: ipc_args},
|
||||
)
|
||||
# Rebind to fresh lists so the yielded chunk's metadata is
|
||||
# not mutated while the consumer is still reading.
|
||||
names, shapes, dtypes, tensor_sizes = [], [], [], []
|
||||
total_bytes = 0
|
||||
|
||||
ipc_buffer[total_bytes : total_bytes + flat.numel()].copy_(flat)
|
||||
names.append(name)
|
||||
shapes.append(list(orig_tensor.shape))
|
||||
dtypes.append(orig_tensor.dtype)
|
||||
tensor_sizes.append(flat.numel())
|
||||
total_bytes += flat.numel()
|
||||
|
||||
if total_bytes:
|
||||
torch.cuda.current_stream().synchronize()
|
||||
yield PackedIpcChunk(
|
||||
names=names,
|
||||
shapes=shapes,
|
||||
dtype_names=[str(d).split(".")[-1] for d in dtypes],
|
||||
tensor_sizes=tensor_sizes,
|
||||
ipc_handle={gpu_uuid: ipc_args},
|
||||
)
|
||||
|
||||
|
||||
class PackedBufferImporter:
|
||||
"""One-slot cache for the consumer-side mapping of a packed IPC buffer.
|
||||
|
||||
torch's cross-process refcount pairs one ``reduce_tensor`` export with
|
||||
one consumer rebuild-release cycle. The producer exports its buffer once
|
||||
and ships the same args with every chunk, so rebuilding and releasing
|
||||
per chunk decrements the counter once per chunk, drives it negative,
|
||||
and the dropped buffer is then never reclaimable by ``ipc_collect`` on
|
||||
the producer side (it leaks one buffer per transfer). Caching the
|
||||
rebuilt buffer keyed by the rebuild args restores the pairing: chunks
|
||||
of one transfer reuse the mapping (it aliases producer memory, so it
|
||||
always reads the current chunk's bytes), and replacing the entry when
|
||||
the next export arrives — or ``close()`` — releases the previous
|
||||
mapping exactly once.
|
||||
|
||||
The importer that consumes a multi-chunk transfer must be one object
|
||||
reused across chunks; a fresh importer per chunk reintroduces the
|
||||
per-chunk release. ``IPCWeightTransferEngine`` owns one instance per
|
||||
engine and closes it on shutdown.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._entry: tuple[tuple, torch.Tensor] | None = None
|
||||
|
||||
def rebuild(self, list_args: list) -> torch.Tensor:
|
||||
from torch.multiprocessing.reductions import rebuild_cuda_tensor
|
||||
|
||||
key = tuple(list_args)
|
||||
if self._entry is not None and self._entry[0] == key:
|
||||
return self._entry[1]
|
||||
packed = rebuild_cuda_tensor(*list_args)
|
||||
self._entry = (key, packed)
|
||||
return packed
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release the held mapping (one producer-side refcount decrement)."""
|
||||
self._entry = None
|
||||
|
||||
|
||||
def packed_ipc_consumer(
|
||||
ipc_handle: dict[str, tuple],
|
||||
names: list[str],
|
||||
shapes: list[list[int]],
|
||||
dtype_names: list[str],
|
||||
tensor_sizes: list[int],
|
||||
device_index: int,
|
||||
importer: "PackedBufferImporter | None" = None,
|
||||
) -> list[tuple[str, torch.Tensor]]:
|
||||
"""Unpack a single packed IPC chunk into named tensors.
|
||||
|
||||
Reconstructs the packed buffer via rebuild_cuda_tensor, unpacks
|
||||
into individual tensors, and clones each into independent storage
|
||||
before returning.
|
||||
|
||||
The clone is intentional: the producer reuses one IPC buffer across
|
||||
chunks, so any tensor view that aliases the buffer would observe the
|
||||
*next* chunk's bytes as soon as the producer's generator is resumed.
|
||||
Callers that retain references past their own update_weights call
|
||||
(notably vLLM's layerwise reload, which buffers ``bound_args`` for
|
||||
replay in ``_layerwise_process``) would otherwise replay against
|
||||
stale data and silently corrupt multi-chunk weight transfers.
|
||||
|
||||
Args:
|
||||
ipc_handle: Mapping of GPU UUID to rebuild_cuda_tensor args tuple
|
||||
names: Parameter names in the packed buffer
|
||||
shapes: Parameter shapes
|
||||
dtype_names: Parameter dtype name strings (e.g. "float16")
|
||||
tensor_sizes: Size in bytes of each parameter in the packed buffer
|
||||
device_index: Local CUDA device index
|
||||
importer: Import cache that must be shared across all chunks of one
|
||||
export so the buffer mapping is opened and released exactly once
|
||||
(see ``PackedBufferImporter``). ``None`` builds an ephemeral one,
|
||||
which is only safe for single-chunk consumption.
|
||||
|
||||
"""
|
||||
props = torch.cuda.get_device_properties(device_index)
|
||||
physical_gpu_id = str(props.uuid)
|
||||
|
||||
if physical_gpu_id not in ipc_handle:
|
||||
raise ValueError(
|
||||
f"IPC handle not found for GPU UUID {physical_gpu_id}. "
|
||||
f"Available UUIDs: {list(ipc_handle.keys())}"
|
||||
)
|
||||
|
||||
args = ipc_handle[physical_gpu_id]
|
||||
list_args = list(args)
|
||||
list_args[6] = device_index
|
||||
if importer is None:
|
||||
importer = PackedBufferImporter()
|
||||
packed = importer.rebuild(list_args)
|
||||
|
||||
content_size = sum(tensor_sizes)
|
||||
packed = packed[:content_size]
|
||||
|
||||
dtypes = [getattr(torch, dn) for dn in dtype_names]
|
||||
return [
|
||||
(name, t.clone())
|
||||
for name, t in unpack_tensor(packed, names, shapes, dtypes, tensor_sizes)
|
||||
]
|
||||
@@ -1,352 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared pieces of the sharded-RDT backend: the op-chain allowlist, buffer
|
||||
sizing and the minimum Ray version, plus ``RdtRouter`` — consumer-only, since
|
||||
routing is a consumer decision.
|
||||
|
||||
The gather-group partition itself is ``base.layerwise_groups``: it defines what a
|
||||
group index means for any ``WeightSource``, not just this transport.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
# The op-chain contract, with two enforcers. The consumer's ``FakeRDTTensor``
|
||||
# intercepts exactly these methods during the bake and records the string name;
|
||||
# anything else reaches ``__torch_dispatch__`` and raises, so a loader needing
|
||||
# real data fails at init rather than transferring wrong bytes. The producer
|
||||
# replays a chain as ``getattr(tensor, op)(...)`` and refuses anything outside
|
||||
# ``ALLOWED_OPS``, so a spoofed consumer cannot invoke arbitrary methods.
|
||||
#
|
||||
# Every entry must be a pure view; ``to`` in particular must never be added, as
|
||||
# it is the dtype/device escape the bake exists to reject. Both sets derive from
|
||||
# this one table so they cannot disagree.
|
||||
SUPPORTED_OPS: dict[Callable, str] = {
|
||||
torch.Tensor.narrow: "narrow",
|
||||
torch.Tensor.view: "view",
|
||||
torch.Tensor.reshape: "reshape",
|
||||
torch.Tensor.__getitem__: "__getitem__",
|
||||
torch.Tensor.unsqueeze: "unsqueeze",
|
||||
torch.Tensor.squeeze: "squeeze",
|
||||
torch.Tensor.transpose: "transpose",
|
||||
torch.Tensor.t: "t",
|
||||
torch.Tensor.permute: "permute",
|
||||
torch.Tensor.flatten: "flatten",
|
||||
torch.Tensor.contiguous: "contiguous",
|
||||
torch.Tensor.chunk: "chunk",
|
||||
# Multi-return like chunk: one child per output, trailing __getitem__(i).
|
||||
torch.Tensor.unbind: "unbind",
|
||||
}
|
||||
|
||||
ALLOWED_OPS = frozenset(SUPPORTED_OPS.values())
|
||||
|
||||
# Ray Direct Transport landed in halves: the actor opt-in
|
||||
# ``enable_tensor_transport`` in 2.49, and ``ray.experimental``'s
|
||||
# ``register_nixl_memory`` / ``set_target_for_ref`` — which this transport calls
|
||||
# directly — in 2.55. The floor is the tested version rather than the earliest
|
||||
# one that could import; 2.55 has never been run against.
|
||||
RDT_MIN_RAY_VERSION = "2.56.0"
|
||||
|
||||
|
||||
def check_ray_rdt_version() -> None:
|
||||
"""Refuse an installed Ray older than the one this backend is tested on.
|
||||
|
||||
vLLM does not depend on Ray, so there is no pin to carry this. Without the
|
||||
check the failure is an opaque option-validation error out of
|
||||
``.options(enable_tensor_transport=True)`` (below 2.49) or an ImportError
|
||||
raised deep in the first pull, long after init reported success (below 2.55).
|
||||
|
||||
Raises:
|
||||
ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``.
|
||||
|
||||
"""
|
||||
import importlib.metadata
|
||||
|
||||
from packaging import version
|
||||
|
||||
required = version.parse(RDT_MIN_RAY_VERSION)
|
||||
current = version.parse(importlib.metadata.version("ray"))
|
||||
if current < required:
|
||||
raise ValueError(
|
||||
f"The 'sharded_rdt' weight transfer backend requires Ray "
|
||||
f">= {required}, the version it is tested against; Ray Direct "
|
||||
f"Transport (ray.experimental.register_nixl_memory / "
|
||||
f"set_target_for_ref) needs at least 2.55. Found {current}. "
|
||||
f"Run `pip install -U 'ray>={required}'`."
|
||||
)
|
||||
|
||||
|
||||
def register_nixl_memory(tensor: torch.Tensor) -> None:
|
||||
"""Register a tensor with Ray using the platform's NIXL package."""
|
||||
from vllm.distributed.nixl_utils import alias_nixl_for_ray
|
||||
|
||||
alias_nixl_for_ray()
|
||||
from ray.experimental import register_nixl_memory as ray_register_nixl_memory
|
||||
|
||||
ray_register_nixl_memory(tensor)
|
||||
|
||||
|
||||
def assign_producer_indices(
|
||||
num_producers: int, num_consumers: int, consumer_idx: int
|
||||
) -> list[int]:
|
||||
"""Producers (global indices) that consumer ``consumer_idx`` binds."""
|
||||
p = max(1, num_producers)
|
||||
c = max(1, num_consumers)
|
||||
if p >= c:
|
||||
return list(range(consumer_idx * p // c, (consumer_idx + 1) * p // c))
|
||||
return [consumer_idx * p // c]
|
||||
|
||||
|
||||
class RdtRouter:
|
||||
"""Decides which producer serves each weight name, and — once bound — sends.
|
||||
|
||||
Ownership is per NAME. ``owner_sets`` holds the few distinct producer sets
|
||||
that occur, and ``name_owner_class[i]`` indexes it for ``names[i]``; a name
|
||||
determines both its owner set and its gather group, so neither appears in the
|
||||
routing API. Empty tables mean every producer holds everything.
|
||||
|
||||
Any placement is expressible this way: a pipeline stage is a set of names
|
||||
sharing one owner set, an expert is a name whose owner set is a single rank,
|
||||
and a group produced by two stages is just two classes inside one group.
|
||||
|
||||
Both engines derive the same tables from the same wire data, so they agree on
|
||||
who serves what. Disagreement is not a wrong answer but a hang or a loud
|
||||
misroute: a pull sent to a producer that never gathered the name trips its
|
||||
served-names guard.
|
||||
|
||||
Routing is consumer-only. The trainer publishes what it holds and answers
|
||||
whatever arrives; it never asks who serves what.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_producers: int,
|
||||
num_consumers: int,
|
||||
owner_sets: list[list[int]] | None = None,
|
||||
name_owner_class: list[int] | None = None,
|
||||
names: list[str] | None = None,
|
||||
group_lens: list[int] | None = None,
|
||||
workers_per_replica: int = 0,
|
||||
) -> None:
|
||||
"""Build the routing tables from the wire data both engines receive.
|
||||
|
||||
The four table arguments travel together and default together: with all
|
||||
of them empty the router degrades to "every producer holds everything".
|
||||
|
||||
Args:
|
||||
num_producers: Trainer ranks. Owner indices are positions in this
|
||||
range, and ``validate`` rejects any that fall outside it.
|
||||
num_consumers: Inference workers across the whole fleet. Fixes
|
||||
the id space; the block carve uses ``workers_per_replica``
|
||||
of it.
|
||||
owner_sets: The distinct producer sets that occur, one row per
|
||||
owner class; each row is deduplicated and sorted here. Empty
|
||||
means a single class owning every producer.
|
||||
name_owner_class: Parallel to ``names`` — ``name_owner_class[i]``
|
||||
indexes ``owner_sets`` for ``names[i]``. A name with no entry
|
||||
falls back to class 0.
|
||||
names: Every parameter name, in group-major order: concatenating
|
||||
the gather groups reproduces this list exactly. The other two
|
||||
tables are keyed by this order.
|
||||
group_lens: Length of each gather group, consecutive over
|
||||
``names``, so they sum to ``len(names)``. Fixes name -> group
|
||||
and the per-group owner union the free barrier fans out to.
|
||||
workers_per_replica: Consumers per inference DEPLOYMENT. The block
|
||||
carve spreads this many consumers over an owner set and every
|
||||
deployment reuses that carve, so the same worker of each
|
||||
deployment resolves one producer (see ``producer_for``). 0
|
||||
means one deployment: carve over the whole fleet.
|
||||
|
||||
Raises:
|
||||
ValueError: ``workers_per_replica`` does not divide
|
||||
``num_consumers``, so the deployments are not uniform and two
|
||||
workers of one deployment would share a block index.
|
||||
|
||||
"""
|
||||
self.num_producers = max(1, num_producers)
|
||||
self.num_consumers = max(1, num_consumers)
|
||||
# Width of the block carve: one deployment's worth of consumers, so
|
||||
# deployments overlay rather than spread. See ``producer_for``.
|
||||
self._block_consumers = min(
|
||||
self.num_consumers, max(1, workers_per_replica or self.num_consumers)
|
||||
)
|
||||
if self.num_consumers % self._block_consumers:
|
||||
raise ValueError(
|
||||
f"workers_per_replica={workers_per_replica} does not divide "
|
||||
f"num_consumers={self.num_consumers}; the inference deployments "
|
||||
f"must be uniform for the per-deployment block carve to be "
|
||||
f"well defined."
|
||||
)
|
||||
self._owner_sets = (
|
||||
[sorted(set(row)) for row in owner_sets]
|
||||
if owner_sets
|
||||
else [list(range(self.num_producers))]
|
||||
)
|
||||
names = list(names or [])
|
||||
classes = list(name_owner_class or [])
|
||||
self._class_of = {
|
||||
n: (classes[i] if i < len(classes) else 0) for i, n in enumerate(names)
|
||||
}
|
||||
# name -> gather group, and the per-group owner union the free barrier
|
||||
# fans out to. Both are pure functions of the tables, so precompute once.
|
||||
self._group_of: dict[str, int] = {}
|
||||
self._group_owners: list[list[int]] = []
|
||||
pos = 0
|
||||
for gi, glen in enumerate(group_lens or []):
|
||||
union: set[int] = set()
|
||||
for n in names[pos : pos + glen]:
|
||||
self._group_of[n] = gi
|
||||
c = self._class_of.get(n, 0)
|
||||
# Bounds-checked: a bad class must surface from validate() as a
|
||||
# named ValueError, not as an IndexError from this precompute.
|
||||
if 0 <= c < len(self._owner_sets):
|
||||
union.update(self._owner_sets[c])
|
||||
pos += glen
|
||||
self._group_owners.append(sorted(union))
|
||||
# Bound at init on the consumer; the rule methods never touch these.
|
||||
self._actors: list[Any] = []
|
||||
self._produce_methods: list[Any] = []
|
||||
self.consumer_id = 0
|
||||
|
||||
# ---------------- rules (pure; safe to use unbound) ----------------
|
||||
|
||||
def class_of(self, name: str) -> int:
|
||||
"""The name's owner class — the planner's bucketing key, since all names
|
||||
of a chunk must share one producer."""
|
||||
return self._class_of.get(name, 0)
|
||||
|
||||
def owners(self, name: str) -> list[int]:
|
||||
"""Every producer holding ``name``."""
|
||||
return list(self._owner_sets[self.class_of(name)])
|
||||
|
||||
def group_owners(self, group_idx: int) -> list[int]:
|
||||
"""Every producer holding any name of ``group_idx``.
|
||||
|
||||
The one group-keyed rule, because the free barrier is per group: a
|
||||
consumer signals ``free_group(gi)`` at each of these and the producer
|
||||
counts signals against its live-consumer total. Names route; groups free.
|
||||
"""
|
||||
if not self._group_owners:
|
||||
return list(range(self.num_producers))
|
||||
return list(self._group_owners[group_idx])
|
||||
|
||||
def producer_for(self, consumer_id: int, name: str) -> int:
|
||||
"""The single producer ``consumer_id`` pulls ``name`` from.
|
||||
|
||||
Blocks consumers across the name's owner set with the same rule that
|
||||
binds producers globally, then rotates by the name's GROUP index so a
|
||||
consumer spreads its groups over its block instead of hammering one NIC.
|
||||
Rotating per group (not per name) keeps every name of a chunk on one
|
||||
producer, which is what lets a chunk be a single pull.
|
||||
|
||||
The carve is over ONE DEPLOYMENT (this consumer's index within its own
|
||||
deployment, over ``workers_per_replica`` of them), so the same worker of
|
||||
every deployment resolves the same producer for every name. Carving over
|
||||
the whole fleet instead spreads the multi-owner names of different
|
||||
deployments onto different producers while single-owner names still
|
||||
route by ownership, so a producer serves several distinct worker
|
||||
indices, each needing its own serve ring, and none of them can share a
|
||||
slot. With one deployment the width is the fleet, so this is the plain
|
||||
block rule.
|
||||
"""
|
||||
own = self.owners(name)
|
||||
if not own:
|
||||
raise ValueError(f"{name!r} has no owner")
|
||||
block = assign_producer_indices(
|
||||
len(own), self._block_consumers, consumer_id % self._block_consumers
|
||||
)
|
||||
return own[block[self._group_of.get(name, 0) % len(block)]]
|
||||
|
||||
def validate(self) -> None:
|
||||
"""Check the ownership tables can be served.
|
||||
|
||||
Raises:
|
||||
ValueError: an owner set is empty or out of range, or a class index
|
||||
does not resolve.
|
||||
|
||||
"""
|
||||
for c, row in enumerate(self._owner_sets):
|
||||
if not row:
|
||||
raise ValueError(f"owner set {c} is empty: no producer holds it")
|
||||
bad = [p for p in row if not 0 <= p < self.num_producers]
|
||||
if bad:
|
||||
raise ValueError(f"owner set {c} out of range: {bad}")
|
||||
for name, c in self._class_of.items():
|
||||
if not 0 <= c < len(self._owner_sets):
|
||||
raise ValueError(
|
||||
f"{name!r} has owner class {c}, but only "
|
||||
f"{len(self._owner_sets)} owner set(s) were shipped"
|
||||
)
|
||||
|
||||
# ---------------- send surface (needs the binding) ----------------
|
||||
|
||||
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
|
||||
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
|
||||
never appended to: every owner index is a position in these lists, so a
|
||||
rejoining engine that re-inits must not shift them."""
|
||||
if len(actors) != len(produce_methods):
|
||||
raise ValueError("actors and produce_methods must be parallel")
|
||||
self._actors = list(actors)
|
||||
self._produce_methods = list(produce_methods)
|
||||
self.consumer_id = consumer_id
|
||||
|
||||
def _bound(self) -> None:
|
||||
if not self._produce_methods:
|
||||
raise RuntimeError("RdtRouter has no producers bound; call bind() first.")
|
||||
|
||||
def pull(self, owner: int, keys: list, seq: int) -> Any:
|
||||
"""Issue one packed pull to ``owner``.
|
||||
|
||||
The consumer id keys the producer's serve ring; without it every worker
|
||||
is served out of ring 0 and concurrent pulls overwrite each other's blob.
|
||||
``seq`` is this call's index in the stream to ``owner`` (see ``_Chunk``):
|
||||
it selects the serve slot in ISSUE order, so a slot is never repacked
|
||||
while the read of its previous contents is still in flight.
|
||||
"""
|
||||
self._bound()
|
||||
return self._produce_methods[owner].remote(
|
||||
keys, consumer_id=self.consumer_id, seq=seq
|
||||
)
|
||||
|
||||
def free_group(self, group_idx: int) -> list:
|
||||
"""Signal the group done at every producer holding any of its names."""
|
||||
self._bound()
|
||||
return [
|
||||
self._actors[p].free_group.remote(group_idx)
|
||||
for p in self.group_owners(group_idx)
|
||||
]
|
||||
|
||||
def reserve_serve_buffers(
|
||||
self, bytes_by_producer: list[int], plan_digests: list[str] | None = None
|
||||
) -> list:
|
||||
"""Ask each producer to pre-register a serve ring sized to the most this
|
||||
consumer will pull from it.
|
||||
|
||||
``plan_digests[p]`` describes the chunks this consumer pulls from
|
||||
producer ``p``, in pull order. A producer that shares one serve ring
|
||||
across the consumers of several deployments compares it across them, so
|
||||
a fleet whose deployments are not identical fails at init instead of
|
||||
stalling mid-sync. A producer that shares nothing ignores it.
|
||||
"""
|
||||
self._bound()
|
||||
digests: list = (
|
||||
list(plan_digests) if plan_digests else [None] * len(bytes_by_producer)
|
||||
)
|
||||
return [
|
||||
self._actors[p].reserve_serve_buffer.remote(
|
||||
self.consumer_id, nb, digests[p]
|
||||
)
|
||||
for p, nb in enumerate(bytes_by_producer)
|
||||
if nb > 0
|
||||
]
|
||||
|
||||
|
||||
def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
|
||||
"""Size a NIXL buffer / ring slot for ``nbytes``: the max of the request, an
|
||||
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
|
||||
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
|
||||
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
|
||||
and the producer's serve rings."""
|
||||
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,300 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Op-chain recording for the sharded-RDT backend.
|
||||
|
||||
The consumer asks the trainer for the exact slice a worker consumes, described as
|
||||
an op chain replayed on the trainer's live tensor. ``FakeRDTTensor`` builds the
|
||||
chain by intercepting the model's own weight loaders; ``copy_`` is its data sink,
|
||||
``BakeSink`` — the dry run records how each slice would be fetched and where it
|
||||
lands.
|
||||
|
||||
Chains are built against meta tensors, so nothing here touches Ray or a GPU.
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_common import SUPPORTED_OPS
|
||||
|
||||
# ("op_name", positional_args, sorted_kwargs_items). Entries must be hashable so
|
||||
# the chain can serve as a FetchKey.
|
||||
OpSpec = tuple[str, tuple[Any, ...], tuple[tuple[str, Any], ...]]
|
||||
OpChain = tuple[OpSpec, ...]
|
||||
FetchKey = tuple[str, OpChain]
|
||||
|
||||
|
||||
def _freeze_kwargs(kwargs: dict[str, Any]) -> tuple[tuple[str, Any], ...]:
|
||||
"""Sort kwargs into a tuple of items for hashable storage in OpSpec."""
|
||||
return tuple(sorted(kwargs.items()))
|
||||
|
||||
|
||||
class _UnsupportedFakeOp(NotImplementedError):
|
||||
"""Raised when a weight loader does something to a FakeRDTTensor that cannot
|
||||
be expressed as a slice request.
|
||||
|
||||
Surfaced as NotImplementedError so callers can distinguish "this backend
|
||||
can't handle this loader" from genuine bugs.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Scatter:
|
||||
"""One recorded scatter: pull ``src``, copy it into ``layer``'s
|
||||
``param_name`` at the recorded strided region. The bake's output unit; see
|
||||
the engine module's Data flow.
|
||||
|
||||
Self-contained so replay needs no lookups. Destination geometry is read off
|
||||
the meta view and rebuilt each sync as ``as_strided(shape, stride, offset)``
|
||||
-- ``layer`` is resolved to a param at replay time, never baked, since every
|
||||
sync re-materializes fresh tensors.
|
||||
|
||||
``shape`` and ``dtype`` also size the slice ON THE WIRE, so ``dtype`` is the
|
||||
PRODUCED dtype (the fake's after its op chain), not the source name's:
|
||||
``view(dtype)`` is allowlisted, and taking the source's would size the slice
|
||||
with the wrong itemsize and carve the packed blob differently on the two
|
||||
sides.
|
||||
"""
|
||||
|
||||
layer: Any
|
||||
param_name: str
|
||||
src: FetchKey
|
||||
offset: int
|
||||
shape: tuple[int, ...]
|
||||
stride: tuple[int, ...]
|
||||
dtype: torch.dtype
|
||||
|
||||
|
||||
def _meta_copy_(dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor:
|
||||
"""Fire ``dest.copy_`` from a zero-storage meta source of ``src``'s geometry.
|
||||
|
||||
Moves no data but still counts against the layer's loaded numel, which drives
|
||||
``_layerwise_process``; skipping it would leave the layer looking unloaded
|
||||
forever.
|
||||
|
||||
A non-meta ``dest`` is one the layerwise reload never moved there (the
|
||||
``SKIP_LOAD_TENSORS`` set, e.g. GLM's router bias). Torch forbids
|
||||
``real.copy_(meta)``, and skipping the count is consistent -- ``get_layer_size``
|
||||
excludes the same set. The caller still RECORDS the copy, so the replay writes
|
||||
the real param.
|
||||
"""
|
||||
if not dest.is_meta:
|
||||
return dest
|
||||
meta_src = torch.empty(src.shape, dtype=src.dtype, device="meta")
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
return dest.copy_(meta_src)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BakeSink:
|
||||
"""``copy_`` sink for the dry-run bake: record how each slice would be fetched
|
||||
and where it would land, and move nothing.
|
||||
|
||||
``_install_recording_stamps`` sets ``current = (leaf_module, param_name)``
|
||||
around each loader so ``accept_copy`` can attribute the copy. An unstamped
|
||||
``copy_`` cannot be attributed and stays unrecorded, so its module fails the
|
||||
coverage gate and takes the plain load. ``copies_by_layer`` is keyed by module
|
||||
object, so iterating it yields each leaf module once.
|
||||
"""
|
||||
|
||||
copies_by_layer: "dict[Any, list[_Scatter | None]]" = field(
|
||||
default_factory=lambda: defaultdict(list)
|
||||
)
|
||||
current: "tuple[Any, str] | None" = None
|
||||
# Source names whose ``copy_`` fired during the bake. Names not here never
|
||||
# moved data (e.g. experts owned by another EP rank) and are skipped; names
|
||||
# here but unbaked fail the plan build.
|
||||
copied_names: "set[str]" = field(default_factory=set)
|
||||
|
||||
def accept_copy(self, dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor:
|
||||
self.copied_names.add(src._name)
|
||||
if self.current is not None:
|
||||
layer, param_name = self.current
|
||||
if dest.numel() != src.numel():
|
||||
# The packed layout sizes each slice from the DESTINATION shape,
|
||||
# while the producer packs what the replayed chain produces. A
|
||||
# broadcasting copy_ makes those two lengths differ, and neither
|
||||
# side can notice: the consumer's buffer view is exactly
|
||||
# prod(dest.shape) elements, so it reshapes cleanly over bytes
|
||||
# the producer laid out at different offsets. Refuse at bake.
|
||||
raise _UnsupportedFakeOp(
|
||||
f"FakeRDTTensor: broadcasting copy_ into {param_name!r} of "
|
||||
f"{type(layer).__name__} ({src.numel()} elements from "
|
||||
f"{src._name!r} into {dest.numel()}). The sharded RDT "
|
||||
"backend sends exactly the slice the loader asks for, so "
|
||||
"the source and destination must have equal numel."
|
||||
)
|
||||
self.copies_by_layer[layer].append(
|
||||
_Scatter(
|
||||
layer=layer,
|
||||
param_name=param_name,
|
||||
src=src._key(),
|
||||
offset=dest.storage_offset(),
|
||||
shape=tuple(dest.shape),
|
||||
stride=tuple(dest.stride()),
|
||||
dtype=src.dtype,
|
||||
)
|
||||
)
|
||||
return _meta_copy_(dest, src)
|
||||
|
||||
|
||||
class FakeRDTTensor(torch.Tensor):
|
||||
"""Zero-storage tensor that records how to fetch a weight slice.
|
||||
|
||||
``_make_wrapper_subclass`` gives it shape/dtype/device without storage. Every
|
||||
op in ``SUPPORTED_OPS`` returns a child with the spec appended; ``copy_``
|
||||
delegates to the installed sink. Anything else reaches ``__torch_dispatch__``
|
||||
and raises ``_UnsupportedFakeOp``, so failures are loud rather than silently
|
||||
fetching the wrong bytes.
|
||||
"""
|
||||
|
||||
# Declared at class scope so mypy can infer attribute types on
|
||||
# instances built via ``_make_wrapper_subclass`` (where ``__new__``
|
||||
# returns a tensor it can't annotate as ``self``).
|
||||
_name: str
|
||||
_ops: OpChain
|
||||
# Handles this fake's copy_. ``None`` only on bare construction.
|
||||
_sink: "BakeSink | None"
|
||||
|
||||
@staticmethod
|
||||
def __new__(
|
||||
cls,
|
||||
name: str,
|
||||
shape: torch.Size,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
ops: OpChain = (),
|
||||
sink: "BakeSink | None" = None,
|
||||
) -> "FakeRDTTensor":
|
||||
t = torch.Tensor._make_wrapper_subclass(
|
||||
cls,
|
||||
shape,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
requires_grad=False,
|
||||
)
|
||||
t._name = name
|
||||
t._ops = tuple(ops)
|
||||
t._sink = sink
|
||||
return t
|
||||
|
||||
def _key(self) -> FetchKey:
|
||||
return (self._name, self._ops)
|
||||
|
||||
def _make_child(
|
||||
self,
|
||||
new_shape: torch.Size,
|
||||
new_dtype: torch.dtype,
|
||||
*new_ops: OpSpec,
|
||||
) -> "FakeRDTTensor":
|
||||
"""Append one or more ops to the chain and return a fresh child.
|
||||
|
||||
Variadic so multi-return ops (e.g. chunk) can append both the base
|
||||
op and an indexing op in a single call.
|
||||
"""
|
||||
return FakeRDTTensor(
|
||||
name=self._name,
|
||||
shape=new_shape,
|
||||
dtype=new_dtype,
|
||||
device=self.device,
|
||||
ops=self._ops + new_ops,
|
||||
sink=self._sink,
|
||||
)
|
||||
|
||||
def _meta(self) -> torch.Tensor:
|
||||
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
|
||||
itself computes post-op geometry rather than us reimplementing it."""
|
||||
return torch.empty(self.shape, dtype=self.dtype, device="meta")
|
||||
|
||||
@classmethod
|
||||
def __torch_function__(
|
||||
cls,
|
||||
func,
|
||||
types,
|
||||
args=(),
|
||||
kwargs=None,
|
||||
):
|
||||
kwargs = kwargs or {}
|
||||
|
||||
# Checked before the allowlist: copy_ must not be recorded into the chain.
|
||||
if func is torch.Tensor.copy_:
|
||||
dest = args[0]
|
||||
src = args[1] if len(args) > 1 else kwargs.get("src")
|
||||
if isinstance(src, cls) and src._sink is not None:
|
||||
return src._sink.accept_copy(dest, src)
|
||||
|
||||
# Allowlisted slice/view/shape ops: append to chain and return child.
|
||||
op_name = SUPPORTED_OPS.get(func)
|
||||
if op_name is not None:
|
||||
self_ = args[0]
|
||||
if isinstance(self_, cls):
|
||||
rest = tuple(args[1:])
|
||||
return cls._intercept(self_, func, op_name, rest, kwargs)
|
||||
|
||||
# Metadata reads are answered by the wrapper subclass and stop here; ops
|
||||
# that need data fall through to __torch_dispatch__, which raises.
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
return func(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def _intercept(
|
||||
cls,
|
||||
self_: "FakeRDTTensor",
|
||||
func: Callable,
|
||||
op_name: str,
|
||||
args: tuple,
|
||||
kwargs: dict,
|
||||
):
|
||||
"""Append the op and return a child, or a tuple of children for
|
||||
multi-return ops. Each child's geometry comes from running the op on a
|
||||
meta tensor."""
|
||||
meta = self_._meta()
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
meta_result = func(meta, *args, **kwargs)
|
||||
|
||||
base_op: OpSpec = (op_name, tuple(args), _freeze_kwargs(kwargs))
|
||||
|
||||
# Single-tensor result: one child carrying base_op alone.
|
||||
if isinstance(meta_result, torch.Tensor):
|
||||
return self_._make_child(meta_result.shape, meta_result.dtype, base_op)
|
||||
|
||||
# Multi-return: one child per output, each carrying base_op plus a
|
||||
# trailing __getitem__(i) so the replay can index the result.
|
||||
if isinstance(meta_result, tuple | list):
|
||||
return tuple(
|
||||
self_._make_child(
|
||||
m.shape,
|
||||
m.dtype,
|
||||
base_op,
|
||||
("__getitem__", (i,), ()),
|
||||
)
|
||||
for i, m in enumerate(meta_result)
|
||||
)
|
||||
|
||||
# Non-tensor result (e.g. .item() snuck into the allowlist).
|
||||
raise _UnsupportedFakeOp(
|
||||
f"FakeRDTTensor: {op_name!r} returned a non-tensor "
|
||||
f"({type(meta_result).__name__}); cannot defer."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __torch_dispatch__(cls, func, types, args=(), kwargs=None):
|
||||
kwargs = kwargs or {}
|
||||
|
||||
# A fake at the aten level means an unsupported loader op. Name the op and
|
||||
# the chain; materializing instead would mask a correctness bug.
|
||||
for arg in (*args, *kwargs.values()):
|
||||
if isinstance(arg, cls):
|
||||
raise _UnsupportedFakeOp(
|
||||
f"FakeRDTTensor: unsupported op {func} reached "
|
||||
f"__torch_dispatch__ on fake {arg._name!r} "
|
||||
f"(chain={arg._ops}). Supported ops are: "
|
||||
f"{sorted(SUPPORTED_OPS.values())}, plus copy_. "
|
||||
"Loaders that need .to(), .float(), .item(), arithmetic, "
|
||||
"bool-mask indexing, or .data access are not supported by "
|
||||
"the sharded RDT backend."
|
||||
)
|
||||
return func(*args, **kwargs)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,377 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sparse NCCL weight transfer engine.
|
||||
|
||||
Sparse patches use checkpoint names, shapes, and flat indices. The model's native
|
||||
weight loader maps them to rank-local runtime parameters, including TP shards and
|
||||
packed parameters.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar, Self
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngine,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
worker_init_payload,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
trainer_init as open_trainer_endpoint,
|
||||
)
|
||||
from vllm.model_executor.model_loader.checkpoint_weight_patch import (
|
||||
CheckpointWeightPatch,
|
||||
load_checkpoint_weight_patches,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SparseWeightPatch",
|
||||
"SparseNCCLTrainerInitInfo",
|
||||
"SparseNCCLWeightTransferUpdateInfo",
|
||||
"SparseNCCLWeightTransferEngine",
|
||||
"SparseNCCLTrainerWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseWeightPatch:
|
||||
"""A sparse patch in checkpoint coordinates."""
|
||||
|
||||
name: str
|
||||
indices: torch.Tensor
|
||||
values: torch.Tensor
|
||||
full_shape: tuple[int, ...]
|
||||
"""Full checkpoint shape."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseNCCLTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for the sparse NCCL weight transfer backend.
|
||||
|
||||
Same rendezvous shape as the dense NCCL backend (the sender opens its
|
||||
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
|
||||
are never packed. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "sparse_nccl"
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Update info for the sparse NCCL weight transfer backend."""
|
||||
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
num_updates_list: list[int]
|
||||
"""Number of sparse entries to receive for each parameter in ``names``."""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
f"`dtype_names` should be of the same size as `names`: "
|
||||
f"got {len(self.dtype_names)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.shapes) != num_params:
|
||||
raise ValueError(
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.num_updates_list) == 0:
|
||||
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
|
||||
if len(self.num_updates_list) != num_params:
|
||||
raise ValueError(
|
||||
f"`num_updates_list` should be of the same size as `names`: "
|
||||
f"got {len(self.num_updates_list)} and {len(self.names)}"
|
||||
)
|
||||
if any(num_updates < 0 for num_updates in self.num_updates_list):
|
||||
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
|
||||
|
||||
|
||||
class SparseNCCLWeightTransferEngine(
|
||||
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
|
||||
):
|
||||
"""Sparse weight transfer engine using NCCL.
|
||||
|
||||
Receives checkpoint-coordinate patches broadcast from the trainer and applies
|
||||
them through the model's native weight loader. Sparse updates modify initialized
|
||||
model tensors in place, so the layerwise reload lifecycle is not used.
|
||||
"""
|
||||
|
||||
init_info_cls = NCCLWeightTransferInitInfo
|
||||
update_info_cls = SparseNCCLWeightTransferUpdateInfo
|
||||
supports_draft_weight_update = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
"""Initialize the NCCL process group with the trainer."""
|
||||
self.model_update_group = worker_init_process_group(
|
||||
init_info, self.parallel_config
|
||||
)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""No-op: sparse patches are applied in place, no layerwise reload."""
|
||||
pass
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""No-op: sparse patches are applied in place, no layerwise reload."""
|
||||
pass
|
||||
|
||||
def receive_weights(self, update_info: SparseNCCLWeightTransferUpdateInfo) -> None:
|
||||
"""Receive sparse flat-index patches from the trainer and apply them."""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
|
||||
# Receive on the communicator's device rather than relying on the
|
||||
# ambient current device in the RPC thread.
|
||||
device = self.model_update_group.device
|
||||
with torch.accelerator.device_index(device.index):
|
||||
stream = torch.cuda.current_stream(device=device)
|
||||
patches = []
|
||||
for name, dtype_name, shape, num_updates in zip(
|
||||
update_info.names,
|
||||
update_info.dtype_names,
|
||||
update_info.shapes,
|
||||
update_info.num_updates_list,
|
||||
strict=True,
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
|
||||
values = torch.empty(num_updates, dtype=dtype, device=device)
|
||||
self.model_update_group.broadcast(indices, src=0, stream=stream)
|
||||
self.model_update_group.broadcast(values, src=0, stream=stream)
|
||||
patches.append(
|
||||
CheckpointWeightPatch(
|
||||
name=name,
|
||||
shape=tuple(shape),
|
||||
dtype=dtype,
|
||||
values=values,
|
||||
indices=indices,
|
||||
)
|
||||
)
|
||||
load_checkpoint_weight_patches(self.model, patches)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
|
||||
|
||||
class SparseNCCLTrainerWeightTransferEngine(
|
||||
TrainerWeightTransferEngine[SparseNCCLTrainerInitInfo]
|
||||
):
|
||||
"""Trainer-side sparse NCCL weight transfer engine.
|
||||
|
||||
Broadcasts flat-index (indices, values) patches from NCCL rank 0 while the
|
||||
inference-side `update_weights` runs concurrently on a side thread (the
|
||||
worker's recvs rendezvous inside the same NCCL broadcasts). `send_weights`
|
||||
owns a complete one-shot lifecycle. RL infrastructure that owns the generic
|
||||
client lifecycle can call `send_weight_chunk` between one start and finish.
|
||||
|
||||
Sparse patches differ every round, so they are not a stable `WeightSource`:
|
||||
the engine takes no `source`, and patches are passed directly to the send
|
||||
methods. An empty patch list is a no-op.
|
||||
|
||||
Only the designated trainer sender joins the transfer group; other trainer
|
||||
ranks skip sparse sends.
|
||||
"""
|
||||
|
||||
init_info_cls = SparseNCCLTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
is_sender: bool = True,
|
||||
) -> None:
|
||||
# Sparse is a delta backend: it takes per-round patches via
|
||||
# send_weights, so a `source` would silently never be sent. The
|
||||
# parameter exists only to match the base/factory signature.
|
||||
if source is not None:
|
||||
raise ValueError(
|
||||
"Sparse NCCL weight transfer takes no WeightSource; pass each "
|
||||
"round's patches to send_weights(patches) instead."
|
||||
)
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: SparseNCCLTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
engine = cls(client=client, source=source, is_sender=init_info.is_sender)
|
||||
if not engine.is_sender:
|
||||
return engine
|
||||
|
||||
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
|
||||
# Sparse transfers are never packed, so the worker keeps the unpacked
|
||||
# defaults on its init info.
|
||||
worker_init_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=1,
|
||||
world_size=init_info.world_size,
|
||||
)
|
||||
|
||||
# The inference workers block inside init_weight_transfer_engine waiting
|
||||
# for the NCCL rendezvous, so we kick that off on a side thread while we
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine,
|
||||
worker_init_payload(worker_init_info),
|
||||
)
|
||||
# Open the trainer endpoint as NCCL rank 0 on the current device
|
||||
# (the init info satisfies the helper's rendezvous protocol).
|
||||
engine.model_update_group = open_trainer_endpoint(init_info)
|
||||
future.result() # surface any inference-side init error
|
||||
|
||||
return engine
|
||||
|
||||
def send_weights(self, patches: Iterable[SparseWeightPatch] | None = None) -> None:
|
||||
"""Broadcast one sparse update through a one-shot lifecycle."""
|
||||
patches = self._prepare_patches(patches)
|
||||
if not patches:
|
||||
return
|
||||
|
||||
self.client.start_weight_update()
|
||||
self._broadcast_chunk(patches)
|
||||
self.client.finish_weight_update()
|
||||
|
||||
def send_weight_chunk(
|
||||
self, patches: Iterable[SparseWeightPatch] | None = None
|
||||
) -> None:
|
||||
"""Broadcast one chunk inside a caller-owned weight update lifecycle."""
|
||||
patches = self._prepare_patches(patches)
|
||||
if not patches:
|
||||
return
|
||||
self._broadcast_chunk(patches)
|
||||
|
||||
def _prepare_patches(
|
||||
self, patches: Iterable[SparseWeightPatch] | None
|
||||
) -> list[SparseWeightPatch]:
|
||||
if not self.is_sender:
|
||||
return []
|
||||
|
||||
prepared = list(patches) if patches is not None else []
|
||||
if not prepared:
|
||||
return []
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError("trainer_init() must be called before sending weights.")
|
||||
for patch in prepared:
|
||||
self._validate_patch(patch)
|
||||
return prepared
|
||||
|
||||
def _broadcast_chunk(self, patches: list[SparseWeightPatch]) -> None:
|
||||
assert self.model_update_group is not None
|
||||
device = self.model_update_group.device
|
||||
device_context = (
|
||||
torch.accelerator.device_index(device.index)
|
||||
if device.type != "cpu"
|
||||
else nullcontext()
|
||||
)
|
||||
with device_context:
|
||||
update_info = SparseNCCLWeightTransferUpdateInfo(
|
||||
names=[patch.name for patch in patches],
|
||||
dtype_names=[
|
||||
str(patch.values.dtype).split(".")[-1] for patch in patches
|
||||
],
|
||||
shapes=[list(patch.full_shape) for patch in patches],
|
||||
num_updates_list=[patch.indices.numel() for patch in patches],
|
||||
)
|
||||
|
||||
# update_weights (workers receive) must run concurrently with the
|
||||
# trainer-side broadcasts — both rendezvous inside the same NCCL calls.
|
||||
executor = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
future = executor.submit(
|
||||
self.client.update_weights, asdict(update_info)
|
||||
)
|
||||
# Surface an RPC that failed before the worker entered NCCL rather
|
||||
# than broadcasting to a peer that will never receive.
|
||||
if future.done():
|
||||
future.result()
|
||||
stream = torch.cuda.current_stream()
|
||||
for patch in patches:
|
||||
self.model_update_group.broadcast(
|
||||
patch.indices, src=0, stream=stream
|
||||
)
|
||||
self.model_update_group.broadcast(
|
||||
patch.values, src=0, stream=stream
|
||||
)
|
||||
future.result() # surface inference-side errors
|
||||
finally:
|
||||
# A failed broadcast can leave the RPC blocked in the matching
|
||||
# receive. Waiting for that thread would hide the original error.
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
self._post_send_sync()
|
||||
|
||||
@staticmethod
|
||||
def _validate_patch(patch: SparseWeightPatch) -> None:
|
||||
"""Reject a malformed patch before starting the NCCL transfer."""
|
||||
if patch.full_shape is None:
|
||||
raise ValueError(f"Sparse patch requires full_shape: {patch.name}")
|
||||
if patch.indices.dtype != torch.int32:
|
||||
raise ValueError(
|
||||
f"Sparse weight updates require int32 indices: {patch.name}"
|
||||
)
|
||||
if patch.indices.ndim != 1 or patch.values.ndim != 1:
|
||||
raise ValueError(
|
||||
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
|
||||
)
|
||||
if patch.indices.numel() != patch.values.numel():
|
||||
raise ValueError(
|
||||
f"`indices` and `values` must have matching lengths for {patch.name}"
|
||||
)
|
||||
|
||||
def _post_send_sync(self) -> None:
|
||||
"""Wait for the broadcasts to land before returning, so a caller may
|
||||
rebuild or free the patch tensors as soon as a send method returns rather
|
||||
than relying on same-stream ordering. See
|
||||
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
|
||||
cross-rank barrier."""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
Reference in New Issue
Block a user