[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
-1422
View File
File diff suppressed because it is too large Load Diff
-8
View File
@@ -43,14 +43,6 @@ CacheDType = Literal[
"fp8",
"fp8_e4m3",
"fp8_e5m2",
"fp8_inc",
"fp8_ds_mla",
"nvfp4_ds_mla",
"turboquant_k8v4",
"turboquant_4bit_nc",
"turboquant_k3v4_nc",
"turboquant_3bit_nc",
"ultraquant_4bit",
"int4_per_token_head",
"int8_per_token_head",
"fp8_per_token_head",
-346
View File
@@ -1,346 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import atexit
import gc
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any
import torch
from vllm.device_allocator import DEFERRABLE_TAGS, AllocationData, HandleType
from vllm.logger import init_logger
from vllm.utils.torch_utils import PIN_MEMORY
logger = init_logger(__name__)
MEMCPY_HOST_TO_DEVICE = 0
MEMCPY_DEVICE_TO_HOST = 1
MEMCPY_DEVICE_TO_DEVICE = 2
xpumem_available = False
xpumem_allocator: Any = None
try:
from vllm_xpu_kernels import xpumem_allocator as _xpumem_allocator
xpumem_allocator = _xpumem_allocator
xpumem_available = True
except ImportError:
xpumem_allocator = None
def _xpu_memory_module() -> Any:
mem_mod = getattr(torch.xpu, "memory", None)
if mem_mod is None:
raise RuntimeError("torch.xpu.memory is not available")
return mem_mod
def _supports_xpu_mem_pool(mem_mod: Any) -> bool:
return hasattr(mem_mod, "MemPool") and hasattr(mem_mod, "use_mem_pool")
def _xpu_memcpy_sync(
dst_ptr: int,
src_ptr: int,
n_bytes: int,
kind: int,
device: int,
) -> None:
def _to_i64_ptr(ptr: int) -> int:
# torch custom-op `int` arguments are signed int64.
# data_ptr() may return a uint64 value above 2^63-1, so normalize it.
return ptr if ptr < (1 << 63) else ptr - (1 << 64)
torch.ops._C.xpu_memcpy_sync(
_to_i64_ptr(dst_ptr),
_to_i64_ptr(src_ptr),
n_bytes,
kind,
device,
)
def get_pluggable_allocator(
python_malloc_fn: Callable[[HandleType], None],
python_free_func: Callable[[int], HandleType],
) -> Any:
if not xpumem_available or xpumem_allocator is None:
raise RuntimeError("xpumem allocator extension is not available")
xpumem_allocator.init_module(python_malloc_fn, python_free_func)
mem_mod = _xpu_memory_module()
alloc_cls = getattr(mem_mod, "XPUPluggableAllocator", None)
if alloc_cls is None:
raise RuntimeError("torch.xpu.memory.XPUPluggableAllocator is not available")
lib_name = xpumem_allocator.__file__
return alloc_cls(lib_name, "my_malloc", "my_free")
def create_and_allocate(allocation_handle: HandleType) -> None:
if not xpumem_available or xpumem_allocator is None:
raise RuntimeError("xpumem allocator extension is not available")
xpumem_allocator.python_create_and_allocate(*allocation_handle)
def unmap_and_release(allocation_handle: HandleType) -> None:
if not xpumem_available or xpumem_allocator is None:
raise RuntimeError("xpumem allocator extension is not available")
xpumem_allocator.python_unmap_and_release(*allocation_handle)
@contextmanager
def use_memory_pool_with_allocator(
python_malloc_fn: Callable[[HandleType], None],
python_free_func: Callable[[int], HandleType],
) -> Iterator[tuple[Any, Any]]:
mem_mod = _xpu_memory_module()
if not _supports_xpu_mem_pool(mem_mod):
raise RuntimeError(
"torch.xpu.memory MemPool APIs are not available "
"(need MemPool and use_mem_pool)."
)
new_alloc = get_pluggable_allocator(python_malloc_fn, python_free_func)
mem_pool = mem_mod.MemPool(new_alloc._allocator)
with mem_mod.use_mem_pool(mem_pool):
yield mem_pool, new_alloc
class XpuMemAllocator:
"""A singleton pluggable allocator helper for XPU.
Note:
Sleep will offload selected payloads to CPU or discard and unmap XPU
physical memory. Wake-up remaps physical memory back to the same
reserved virtual address and restores payload.
"""
instance: "XpuMemAllocator | None" = None
default_tag: str = "default"
@staticmethod
def get_instance() -> "XpuMemAllocator":
assert xpumem_available, "xpumem allocator is not available"
if XpuMemAllocator.instance is None:
XpuMemAllocator.instance = XpuMemAllocator()
# Ensure MemPool/allocator wrappers are released before interpreter
# finalization tears down XPU runtime internals.
atexit.register(XpuMemAllocator._shutdown_singleton)
return XpuMemAllocator.instance
@staticmethod
def _shutdown_singleton() -> None:
instance = XpuMemAllocator.instance
if instance is None:
return
try:
instance.release_pools()
except Exception:
logger.exception("XpuMemAllocator singleton shutdown failed")
def __init__(self):
self.pointer_to_data: dict[int, AllocationData] = {}
self.current_tag: str = XpuMemAllocator.default_tag
self.allocator_and_pools: dict[str, list[Any]] = {}
self._pending_allocators: list[Any] = []
self.python_malloc_callback = self._python_malloc_callback
self.python_free_callback = self._python_free_callback
def _python_malloc_callback(self, allocation_handle: HandleType) -> None:
ptr = allocation_handle[2]
self.pointer_to_data[ptr] = AllocationData(allocation_handle, self.current_tag)
logger.debug(
"Allocated %s bytes for %s at %s",
allocation_handle[1],
self.current_tag,
ptr,
)
def _python_free_callback(self, ptr: int) -> HandleType:
data = self.pointer_to_data.pop(ptr)
data.cpu_backup_tensor = None
logger.debug("Freed %s bytes for %s at %s", data.handle[1], data.tag, ptr)
return data.handle
def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None:
if offload_tags is None:
offload_tags = (XpuMemAllocator.default_tag,)
elif isinstance(offload_tags, str):
offload_tags = (offload_tags,)
assert isinstance(offload_tags, tuple)
total_bytes = 0
backup_bytes = 0
has_policy_conflict = False
for ptr, data in self.pointer_to_data.items():
if data.is_asleep:
requests_offload = data.tag in offload_tags
was_offloaded = data.cpu_backup_tensor is not None
if requests_offload != was_offloaded:
has_policy_conflict = True
continue
size_in_bytes = data.handle[1]
total_bytes += size_in_bytes
if data.tag not in offload_tags:
unmap_and_release(data.handle)
data.is_asleep = True
continue
backup_bytes += size_in_bytes
device, _, _, _ = data.handle
cpu_backup_tensor = torch.empty(
size_in_bytes,
dtype=torch.uint8,
device="cpu",
pin_memory=PIN_MEMORY,
)
cpu_ptr = cpu_backup_tensor.data_ptr()
_xpu_memcpy_sync(
cpu_ptr,
ptr,
size_in_bytes,
MEMCPY_DEVICE_TO_HOST,
device,
)
data.cpu_backup_tensor = cpu_backup_tensor
unmap_and_release(data.handle)
data.is_asleep = True
logger.info(
"XpuMemAllocator: sleep freed %.2f GiB memory in total, of which "
"%.2f GiB is backed up in CPU and the rest %.2f GiB is discarded "
"directly.",
total_bytes / 1024**3,
backup_bytes / 1024**3,
(total_bytes - backup_bytes) / 1024**3,
)
if has_policy_conflict:
logger.warning(
"XpuMemAllocator: sleep cannot change the policy of "
"already-asleep allocations; the existing policy was kept."
)
gc.collect()
xpu_empty_cache = getattr(torch.xpu, "empty_cache", None)
if callable(xpu_empty_cache):
xpu_empty_cache()
def discard(self, tags: tuple[str, ...] | str) -> None:
"""Discard mapped allocations with the given tags without CPU backup."""
if isinstance(tags, str):
tags = (tags,)
discarded_bytes = 0
has_policy_conflict = False
for data in self.pointer_to_data.values():
if data.tag not in tags:
continue
if data.is_asleep:
if data.cpu_backup_tensor is not None:
has_policy_conflict = True
continue
torch.accelerator.synchronize(data.handle[0])
unmap_and_release(data.handle)
data.is_asleep = True
discarded_bytes += data.handle[1]
logger.info(
"XpuMemAllocator: discarded %.2f GiB for tags %s.",
discarded_bytes / 1024**3,
tags,
)
if has_policy_conflict:
logger.warning(
"XpuMemAllocator: discard cannot change the policy of "
"already-asleep allocations; the existing policy was kept."
)
def wake_up(self, tags: list[str] | None = None) -> None:
for ptr, data in self.pointer_to_data.items():
if not data.is_asleep:
continue
if (
tags is not None
and data.tag not in tags
and data.tag in DEFERRABLE_TAGS
):
continue
create_and_allocate(data.handle)
data.is_asleep = False
cpu_backup_tensor = data.cpu_backup_tensor
if cpu_backup_tensor is None:
continue
device, size_in_bytes, _, _ = data.handle
_xpu_memcpy_sync(
ptr,
cpu_backup_tensor.data_ptr(),
size_in_bytes,
MEMCPY_HOST_TO_DEVICE,
device,
)
data.cpu_backup_tensor = None
def release_pools(self) -> None:
"""Release pools, retaining allocators until their allocations are freed."""
if not self.allocator_and_pools and not self._pending_allocators:
return
pool_entries = [e for es in self.allocator_and_pools.values() for e in es]
self.allocator_and_pools.clear()
mem_pools = [entry[0] for entry in pool_entries]
# Live tensors can outlast their MemPool wrappers. Keep allocators alive
# across calls so deferred frees cannot invoke a destroyed allocator.
self._pending_allocators.extend(entry[1] for entry in pool_entries)
pool_entries.clear()
xpu_sync = getattr(torch.xpu, "synchronize", None)
if callable(xpu_sync):
try:
xpu_sync()
except Exception:
logger.debug("torch.xpu.synchronize() failed during release_pools")
mem_pools.clear()
gc.collect()
torch.xpu.empty_cache()
# Free callbacks remove allocation records, including cached blocks.
# If any remain, a later release_pools() call must retry this cleanup.
if not self.pointer_to_data:
self._pending_allocators.clear()
@contextmanager
def use_memory_pool(self, tag: str | None = None):
if tag is None:
tag = XpuMemAllocator.default_tag
old_tag = self.current_tag
self.current_tag = tag
try:
with use_memory_pool_with_allocator(
self.python_malloc_callback,
self.python_free_callback,
) as data:
# Keep every entry's pool alive (pytorch/pytorch#146431).
self.allocator_and_pools.setdefault(tag, []).append(data)
yield
finally:
self.current_tag = old_tag
def get_current_usage(self) -> int:
total = 0
for data in self.pointer_to_data.values():
total += data.handle[1]
return total
@@ -1,296 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup
import vllm.envs as envs
from vllm.logger import init_logger
from .base_device_communicator import DeviceCommunicatorBase
logger = init_logger(__name__)
class XpuCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
use_all2all: bool = False,
):
super().__init__(
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
)
self.ca_comm: None = None
if self.use_all2all:
if self.all2all_backend in ("naive", "allgather_reducescatter"):
from .all2all import AgRsAll2AllManager
self.all2all_manager = AgRsAll2AllManager(self.cpu_group)
logger.info("Using AgRs manager on XPU device.")
else: # type: ignore[has-type]
logger.warning(
"`%s` all2all manager is not supported on XPU. "
"Falling back to AgRs manager for XPU, "
"which is the Default backend",
self.all2all_backend, # type: ignore[has-type]
)
from .all2all import AgRsAll2AllManager
self.all2all_manager = AgRsAll2AllManager(self.cpu_group)
logger.info("Using AgRs manager on XPU device.")
def _fixed_rank_sum(self, input_: torch.Tensor) -> torch.Tensor:
flat_input = input_.reshape(-1)
gathered = torch.empty(
(self.world_size, flat_input.numel()),
dtype=input_.dtype,
device=input_.device,
)
dist.all_gather_single(gathered.view(-1), flat_input, group=self.device_group)
output = gathered[0].clone()
for rank in range(1, self.world_size):
output.add_(gathered[rank])
return output.view(input_.shape)
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
if envs.VLLM_BATCH_INVARIANT:
# XCCL changes its reduction order with message size. Gather first
# and accumulate in rank order, independent of the token batch.
return self._fixed_rank_sum(input_)
output = input_.clone()
dist.all_reduce(output, group=self.device_group)
return output
def reduce_scatter(self, input_: torch.Tensor, dim: int = -1):
world_size = self.world_size
if dim < 0:
# Convert negative dim to positive.
dim += input_.dim()
# Note: This will produce an incorrect answer if we don't make
# the input_tensor contiguous. Possible bug in reduce_scatter_tensor?
input_tensor = input_.movedim(0, dim).contiguous()
assert input_tensor.shape[0] % world_size == 0
chunk_size = input_tensor.shape[0] // world_size
output_shape = (chunk_size,) + input_tensor.shape[1:]
output = torch.empty(
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
)
if envs.VLLM_BATCH_INVARIANT:
reduced = self._fixed_rank_sum(input_tensor)
start = self.rank_in_group * chunk_size
output.copy_(reduced.narrow(0, start, chunk_size))
else:
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
# Reshape before returning
return output.movedim(0, dim).contiguous()
def reduce_scatterv(
self, input_: torch.Tensor, dim: int = -1, sizes: list[int] | None = None
):
world_size = self.world_size
if dim < 0:
# Convert negative dim to positive.
dim += input_.dim()
# Note: This will produce an incorrect answer if we don't make
# the input_tensor contiguous. Possible bug in reduce_scatter_tensor?
input_tensor = input_.movedim(0, dim).contiguous()
if sizes is not None:
assert len(sizes) == world_size
assert input_tensor.shape[0] == sum(sizes)
chunk_size = sizes[self.rank_in_group]
else:
assert input_tensor.shape[0] % world_size == 0
chunk_size = input_tensor.shape[0] // world_size
output_shape = (chunk_size,) + input_tensor.shape[1:]
output = torch.empty(
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
)
if envs.VLLM_BATCH_INVARIANT:
reduced = self._fixed_rank_sum(input_tensor)
start = (
self.rank_in_group * chunk_size
if sizes is None
else sum(sizes[: self.rank_in_group])
)
output.copy_(reduced.narrow(0, start, chunk_size))
elif sizes is None or sizes.count(sizes[0]) == len(sizes):
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
elif torch.xpu.device_count() < self.world_size:
# uneven reduce_scatter is not supported when the
# process cannot see every device in the group (
# e.g. external-LBDP).
max_size = max(sizes)
padded = input_tensor.new_zeros(
(world_size * max_size,) + input_tensor.shape[1:]
)
for rank, split in enumerate(input_tensor.split(sizes, dim=0)):
padded[rank * max_size : rank * max_size + sizes[rank]] = split
padded_output = input_tensor.new_empty((max_size,) + input_tensor.shape[1:])
dist.reduce_scatter_tensor(padded_output, padded, group=self.device_group)
output = padded_output[:chunk_size]
else:
input_splits = list(input_tensor.split(sizes, dim=0))
dist.reduce_scatter(output, input_splits, group=self.device_group)
# Reshape before returning
return output.movedim(0, dim).contiguous()
def all_gatherv(
self,
input_: torch.Tensor | list[torch.Tensor],
dim: int = 0,
sizes: list[int] | None = None,
):
if dim != 0:
raise NotImplementedError("only dim 0 all-gatherv is supported")
world_size = self.world_size
# 'sizes' is not needed if all inputs in the same group have the same
# shape
if sizes is not None and all(s == sizes[0] for s in sizes):
sizes = None
def _all_gather_single(input_: torch.Tensor, sizes: list[int] | None = None):
input_size = input_.size()
if sizes is not None:
assert len(sizes) == world_size
assert input_.shape[dim] == sizes[self.rank_in_group], (
f"{input_.shape[dim]} != {sizes[self.rank_in_group]}"
)
output_size = (sum(sizes),) + input_size[1:]
else:
output_size = (input_size[0] * world_size,) + input_size[1:]
# Allocate output tensor.
output_tensor = torch.empty(
output_size, dtype=input_.dtype, device=input_.device
)
if sizes is not None:
all_gather_list = []
for size in sizes:
all_gather_list.append(
torch.empty(
(size,) + input_.shape[1:],
dtype=input_.dtype,
device=input_.device,
)
)
dist.all_gather(all_gather_list, input_, group=self.device_group)
output_tensor = torch.cat(all_gather_list, dim=0)
else:
dist.all_gather([output_tensor], input_, group=self.device_group)
return output_tensor
if isinstance(input_, torch.Tensor):
return _all_gather_single(input_, sizes)
output_list = []
for inp in input_:
output_list.append(_all_gather_single(inp, sizes=sizes))
return output_list
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> torch.Tensor | None:
assert -input_.dim() <= dim < input_.dim(), (
f"Invalid dim ({dim}) for input tensor with shape {input_.size()}"
)
if dim < 0:
# Convert negative dim to positive.
dim += input_.dim()
# For xpu path, gather doesn't work properly together with ray
# cluster so we use all_gather instead for now.
input_size = input_.size()
# Allocate output tensor.
output_tensor = torch.empty(
(self.world_size,) + input_size, dtype=input_.dtype, device=input_.device
)
# All-gather.
dist.all_gather_single(output_tensor, input_, group=self.device_group)
if self.rank_in_group == dst:
# Reshape
output_tensor = output_tensor.movedim(0, dim)
output_tensor = output_tensor.reshape(
input_size[:dim]
+ (self.world_size * input_size[dim],)
+ input_size[dim + 1 :]
)
else:
output_tensor = None
return output_tensor
def broadcast(self, input_: torch.Tensor, src: int = 0) -> None:
dist.broadcast(input_, src=src, group=self.device_group)
def dispatch_router_logits(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
is_sequence_parallel: bool = False,
extra_tensors: list[torch.Tensor] | None = None,
) -> (
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
router_logits,
is_sequence_parallel,
extra_tensors,
)
def dispatch(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
is_sequence_parallel: bool = False,
extra_tensors: list[torch.Tensor] | None = None,
) -> (
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch(
hidden_states,
topk_weights,
topk_ids,
is_sequence_parallel,
extra_tensors=extra_tensors,
)
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.combine(
hidden_states,
is_sequence_parallel,
)
@@ -1,9 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Internal building blocks for the Mooncake encoder-cache connector.
The public connector remains in :mod:`mooncake_ec_connector`. This package
separates configuration, control-plane messaging, transfer state, registered
memory, and worker/scheduler orchestration so that each component has one
owner and can be tested independently.
"""
@@ -1,155 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING
from vllm.utils.network_utils import make_zmq_path
if TYPE_CHECKING:
from vllm.config import VllmConfig
# Scheduler and Worker must agree on when a remote reservation becomes stale.
_RESERVATION_TTL_SECONDS = 300
def _positive_int(name: str, value: object) -> int:
message = f"ECMooncakeConnector requires {name} to be a positive integer."
if isinstance(value, bool):
raise ValueError(message)
if isinstance(value, int):
result = value
elif isinstance(value, float) and math.isfinite(value) and value.is_integer():
result = int(value)
elif isinstance(value, str):
try:
result = int(value)
except ValueError as error:
raise ValueError(message) from error
else:
raise ValueError(message)
if result <= 0:
raise ValueError(f"ECMooncakeConnector requires {name} > 0.")
return result
def _positive_float(name: str, value: object) -> float:
message = f"ECMooncakeConnector requires {name} > 0."
if isinstance(value, bool):
raise ValueError(message)
try:
result = float(value) # type: ignore[arg-type]
except (TypeError, ValueError) as error:
raise ValueError(message) from error
if not math.isfinite(result) or result <= 0:
raise ValueError(message)
return result
@dataclass(frozen=True)
class MooncakeECConfig:
"""Validated settings shared by the Scheduler and Worker roles.
``control_port`` is the first TP-shard port after the DP offset;
``control_addr`` targets that shard, which advertises the full topology.
``control_host`` is the address this instance both advertises and binds,
so reaching a Consumer means being routed to it rather than finding it on
every interface.
"""
is_producer: bool
is_consumer: bool
protocol: str
buffer_device: str
control_host: str
control_port: int
control_addr: str
control_timeout_ms: int
push_wait_timeout_s: float
pool_size: int
cross_encoder_cache: bool = False
store_max_pending_items: int = 32
store_max_pending_bytes: int = 2 * 1024**3
store_read_buffer_bytes: int = 128 * 1024**2
@classmethod
def from_vllm_config(cls, vllm_config: VllmConfig) -> MooncakeECConfig:
parallel_config = vllm_config.parallel_config
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
if ec_config.is_ec_producer:
if parallel_config.tensor_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers require tensor_parallel_size=1."
)
if parallel_config.pipeline_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers do not support pipeline parallelism."
)
if parallel_config.data_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers require data_parallel_size=1."
)
registered_buffer_size = _positive_int(
"ec_buffer_size", ec_config.ec_buffer_size
)
get = ec_config.get_from_extra_config
shared_reuse = get("cross_encoder_cache", False)
if not isinstance(shared_reuse, bool):
raise ValueError("cross_encoder_cache must be a boolean")
if shared_reuse:
if not ec_config.is_encode_only:
raise ValueError(
"cross_encoder_cache requires an Encoder-only producer"
)
if not vllm_config.use_v2_model_runner:
raise ValueError("cross_encoder_cache requires Model Runner V2")
if vllm_config.lora_config is not None:
raise ValueError("cross_encoder_cache does not support LoRA")
mm_config = vllm_config.model_config.multimodal_config
if mm_config is not None and mm_config.mm_processor_cache_gb == 0:
raise ValueError(
"cross_encoder_cache requires mm_processor_cache_gb > 0 "
"to preserve content identifiers across Encoders and restarts"
)
control_port = int(ec_config.ec_port) + (
parallel_config.data_parallel_index * parallel_config.tensor_parallel_size
)
highest_port = control_port + parallel_config.tensor_parallel_size - 1
if not 1 <= control_port <= highest_port <= 65535:
raise ValueError("ECMooncakeConnector ec_port must be in 1..65535.")
return cls(
is_producer=ec_config.is_ec_producer,
is_consumer=ec_config.is_ec_consumer,
protocol=str(get("mooncake_protocol", "rdma")),
buffer_device=str(ec_config.ec_buffer_device or "cuda").lower(),
control_host=str(ec_config.ec_ip),
control_port=control_port,
control_addr=make_zmq_path("tcp", ec_config.ec_ip, control_port),
control_timeout_ms=max(
1,
math.ceil(
_positive_float("control_timeout_s", get("control_timeout_s", 30))
* 1000
),
),
push_wait_timeout_s=_positive_float(
"push_wait_timeout_s", get("push_wait_timeout_s", 60)
),
pool_size=registered_buffer_size,
cross_encoder_cache=shared_reuse,
store_max_pending_items=_positive_int(
"store_max_pending_items", get("store_max_pending_items", 32)
),
store_max_pending_bytes=_positive_int(
"store_max_pending_bytes", get("store_max_pending_bytes", 2 * 1024**3)
),
store_read_buffer_bytes=_positive_int(
"store_read_buffer_bytes", get("store_read_buffer_bytes", 128 * 1024**2)
),
)
@@ -1,460 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ZMQ control plane for Mooncake encoder-cache reservations and events.
Tensor bytes never travel through this module. It coordinates destination
reservations, completion/cancellation, TP-shard discovery, and readiness
notifications while :mod:`transfer` owns the Mooncake data plane.
"""
from __future__ import annotations
import threading
import time
from collections import deque
from collections.abc import Callable
from concurrent.futures import Executor, Future
from typing import Any
import torch
import zmq
from vllm.logger import init_logger
from vllm.utils.network_utils import make_zmq_path
logger = init_logger(__name__)
_MAX_PENDING_EVENTS = 4096
_RESERVATION_REAP_INTERVAL_SECONDS = 1
ControlRequest = dict[str, Any]
ControlResponse = dict[str, Any]
class ControlClient:
"""Send control requests through reusable, thread-local REQ sockets."""
def __init__(self, timeout_ms: int) -> None:
self._context = zmq.Context()
self._timeout_ms = timeout_ms
self._local = threading.local()
self._topologies: dict[str, list[str]] = {}
self._closed = False
def _sockets(self) -> dict[str, zmq.Socket]:
sockets = getattr(self._local, "sockets", None)
if sockets is None:
sockets = {}
self._local.sockets = sockets
return sockets
def _discard(self, addr: str) -> None:
socket = self._sockets().pop(addr, None)
if socket is not None:
socket.close(linger=0)
def _exchange(self, addr: str, payload: ControlRequest) -> ControlResponse:
sockets = self._sockets()
socket = sockets.get(addr)
if socket is None:
socket = self._context.socket(zmq.REQ)
socket.setsockopt(zmq.IPV6, 1)
socket.setsockopt(zmq.RCVTIMEO, self._timeout_ms)
socket.setsockopt(zmq.SNDTIMEO, self._timeout_ms)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(addr)
sockets[addr] = socket
try:
socket.send_json(payload)
response = socket.recv_json()
except Exception:
self._discard(addr)
raise
assert isinstance(response, dict)
return response
def request(self, addr: str, payload: ControlRequest) -> Any:
response = self._exchange(addr, payload)
if not response.get("ok"):
raise RuntimeError(response.get("error", "EC control request failed"))
return response.get("result")
def discover_shards(self, base_addr: str) -> list[str] | None:
if base_addr in self._topologies:
return self._topologies[base_addr]
try:
reply = self.request(base_addr, {"op": "peers"})
ports = reply.get("ports") if isinstance(reply, dict) else None
if not isinstance(ports, list) or not ports:
raise ValueError("invalid or empty peer list")
prefix = base_addr.rsplit(":", 1)[0]
shards = [f"{prefix}:{int(port)}" for port in ports]
except Exception:
logger.warning(
"EC Mooncake consumer at %s did not report its shards",
base_addr,
exc_info=True,
)
return None
self._topologies[base_addr] = shards
return shards
def close(self) -> None:
if self._closed:
return
self._closed = True
self._context.destroy(linger=0)
def make_cancel_request(
transfer_id: str,
reservation_id: str = "",
*,
abandon: bool = False,
refresh: bool = False,
) -> ControlRequest:
request: ControlRequest = {
"op": "cancel",
"transfer_id": transfer_id,
"reservation_id": reservation_id,
}
if abandon:
request["abandon"] = True
if refresh:
request["refresh"] = True
return request
class EventInbox:
"""Receive Consumer readiness events without blocking the Scheduler.
Subscribing needs one blocking control request per shard, so it runs on
`executor` when one is given: an unreachable Consumer must cost the
Scheduler nothing, not one control timeout per drain.
"""
def __init__(self, client: ControlClient, executor: Executor | None = None) -> None:
self._client = client
self._executor = executor
self._discovery: Future[list[str] | None] | None = None
self._context: zmq.Context | None = None
self._socket: zmq.Socket | None = None
self._closed = False
self.shard_count = 1
def _endpoints(self, base_addr: str) -> list[str] | None:
"""Ask every shard where it publishes readiness events.
Blocking: called on `self._executor` unless there is none.
"""
shards = self._client.discover_shards(base_addr)
if shards is None:
return None
endpoints = []
for addr in shards:
try:
event_port = self._client.request(addr, {"op": "event_port"})
address, _ = addr.rsplit(":", 1)
endpoints.append(f"{address}:{int(event_port)}")
except Exception:
logger.warning(
"EC Mooncake could not subscribe to the event channel of "
"consumer shard %s; retrying the complete topology later.",
addr,
)
return None
return endpoints
def _connect(self, base_addr: str) -> None:
if self._socket is not None or self._closed:
return
if self._executor is None:
self._install(self._endpoints(base_addr))
return
if self._discovery is None:
self._discovery = self._executor.submit(self._endpoints, base_addr)
return
if not self._discovery.done():
return
discovery, self._discovery = self._discovery, None
try:
self._install(discovery.result())
except Exception:
logger.warning(
"EC Mooncake event-channel discovery for %s failed; retrying.",
base_addr,
exc_info=True,
)
def _install(self, endpoints: list[str] | None) -> None:
"""Adopt a discovered topology.
Runs on the caller's thread so the socket is only ever touched there.
"""
if not endpoints or self._closed:
return
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.setsockopt(zmq.IPV6, 1)
for endpoint in endpoints:
socket.connect(endpoint)
self._context = context
self._socket = socket
self.shard_count = len(endpoints)
def drain(self, base_addr: str) -> list[dict[str, Any]]:
self._connect(base_addr)
if self._socket is None:
return []
events = []
while True:
try:
events.append(self._socket.recv_json(flags=zmq.DONTWAIT))
except zmq.Again:
return events
except zmq.ZMQError:
logger.warning("EC Mooncake event channel failed", exc_info=True)
return events
except (ValueError, UnicodeError):
# The event channel is a plain PULL socket: an undecodable
# frame must cost one frame, not the engine.
logger.warning(
"Discarding an undecodable EC Mooncake event.", exc_info=True
)
def close(self) -> None:
if self._closed:
return
self._closed = True
if self._discovery is not None:
self._discovery.cancel()
self._discovery = None
if self._socket is not None:
self._socket.close(linger=0)
self._socket = None
if self._context is not None:
self._context.term()
class ConsumerControlServer:
"""Expose Consumer reservations and readiness events over ZMQ.
One server runs on every receiving TP rank. The REP channel handles
reservation operations, while a PUSH channel publishes newly ready items
to the Scheduler.
"""
def __init__(
self,
host: str,
port: int,
reserve: Callable[[dict[str, Any]], dict[str, Any]],
status: Callable[[str], dict[str, Any] | None],
complete: Callable[[str, str], tuple[bool, bool]],
cancel: Callable[[str, str, bool, bool], bool],
reap: Callable[[], int],
peer_ports: list[int] | None = None,
device: torch.device | None = None,
drain_ready: Callable[[], list[str]] = lambda: [],
) -> None:
self.host = host
self.port = port
self.peer_ports = peer_ports or [port]
self._device = device
self.event_port: int | None = None
self._reserve = reserve
self._status = status
self._complete = complete
self._cancel = cancel
self._reap = reap
self._drain_ready = drain_ready
self._stop = threading.Event()
self._started = threading.Event()
self._thread: threading.Thread | None = None
self._startup_error: Exception | None = None
def start(self) -> None:
def loop() -> None:
if self._device is not None and self._device.type == "cuda":
torch.accelerator.set_device_index(self._device.index or 0)
context = zmq.Context()
socket = context.socket(zmq.REP)
event_socket = context.socket(zmq.PUSH)
socket.setsockopt(zmq.IPV6, 1)
event_socket.setsockopt(zmq.IPV6, 1)
pending_events: deque[dict[str, Any]] = deque()
def queue_event(event: dict[str, Any]) -> None:
event["shard"] = self.port
if len(pending_events) >= _MAX_PENDING_EVENTS:
dropped = pending_events.popleft()
logger.warning(
"EC Mooncake event backlog full on port %d; dropping "
"readiness for transfer_id=%s",
self.port,
dropped.get("transfer_id"),
)
pending_events.append(event)
def queue_ready(transfer_id: str) -> None:
status = self._status(transfer_id)
if status is not None:
queue_event({"transfer_id": transfer_id, **status})
last_reap_at = time.monotonic()
socket.setsockopt(zmq.RCVTIMEO, 100)
try:
socket.bind(make_zmq_path("tcp", self.host, self.port))
event_socket.bind(make_zmq_path("tcp", self.host, 0))
self.event_port = int(
event_socket.getsockopt(zmq.LAST_ENDPOINT)
.decode()
.rsplit(":", 1)[1]
)
except Exception as e:
self._startup_error = e
self._started.set()
socket.close(linger=0)
event_socket.close(linger=0)
context.term()
return
self._started.set()
try:
while not self._stop.is_set():
while pending_events:
try:
event_socket.send_json(
pending_events[0], flags=zmq.DONTWAIT
)
except zmq.Again:
break
pending_events.popleft()
now = time.monotonic()
if now - last_reap_at >= _RESERVATION_REAP_INTERVAL_SECONDS:
self._reap()
last_reap_at = now
try:
request = socket.recv_json()
except zmq.Again:
continue
except Exception:
# The frame arrived but did not decode. REP still owes a
# reply, so answer before returning to the loop.
logger.exception(
"EC Mooncake control channel on port %d received an "
"undecodable request",
self.port,
)
socket.send_json(
{"ok": False, "error": "malformed control request"}
)
continue
try:
op = request.get("op")
result: Any = None
if op in ("reserve", "reserve_batch"):
items = (
request["items"] if op == "reserve_batch" else [request]
)
results = []
for item in items:
try:
reserved = self._reserve(item)
results.append({"ok": True, "result": reserved})
if reserved.get("ready"):
queue_ready(str(item["transfer_id"]))
except Exception as exc:
results.append({"ok": False, "error": str(exc)})
if op == "reserve_batch":
result = {"items": results}
elif not results[0]["ok"]:
raise RuntimeError(results[0]["error"])
else:
result = results[0]["result"]
elif op == "status":
result = self._status(str(request["transfer_id"]))
elif op == "event_port":
result = self.event_port
elif op == "peers":
result = {"ports": self.peer_ports}
elif op in ("complete", "complete_batch"):
items = (
request["items"]
if op == "complete_batch"
else [request]
)
completions = []
for item in items:
transfer_id = str(item["transfer_id"])
accepted, became_ready = self._complete(
transfer_id, str(item["reservation_id"])
)
for ready_id in self._drain_ready():
queue_ready(ready_id)
completions.append(
{
"completed": accepted,
"became_ready": became_ready,
}
)
if not became_ready:
continue
queue_ready(transfer_id)
result = (
{"items": completions}
if op == "complete_batch"
else completions[0]
)
elif op == "cancel":
result = {
"cancelled": self._cancel(
str(request["transfer_id"]),
str(request.get("reservation_id", "")),
bool(request.get("abandon", False)),
bool(request.get("refresh", False)),
)
}
else:
raise ValueError(f"unknown control op: {op!r}")
socket.send_json({"ok": True, "result": result})
except Exception as e:
socket.send_json({"ok": False, "error": str(e)})
except Exception:
# `finally` closes the sockets and the thread ends, so without
# this every later reserve against this shard would surface
# only as a control timeout with nothing to attribute it to.
logger.exception(
"EC Mooncake control channel on port %d stopped serving",
self.port,
)
raise
finally:
socket.close(linger=0)
event_socket.close(linger=0)
context.term()
self._thread = threading.Thread(
target=loop, name="ec-mooncake-control", daemon=True
)
self._thread.start()
if not self._started.wait(timeout=5):
raise RuntimeError("EC Mooncake control channel failed to start")
if self._startup_error is not None:
raise RuntimeError("EC Mooncake control channel failed to bind") from (
self._startup_error
)
logger.info(
"EC Mooncake control channel listening on tcp://%s:%d (events tcp://%s:%d)",
self.host,
self.port,
self.host,
self.event_port,
)
def close(self) -> None:
if self._thread is None:
return
self._stop.set()
self._thread.join()
self._thread = None
@@ -1,504 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Registered-memory allocation and residency for Mooncake transfers.
The Producer pool stages source tensors in a registered slab. The Consumer
pool owns destination allocations from reservation through publication,
resident reuse, CUDA-safe retirement, and pressure-driven reclamation.
"""
from __future__ import annotations
import bisect
import threading
from collections import OrderedDict
from collections.abc import Callable
from dataclasses import dataclass
from typing import Generic, TypeVar
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
MooncakeTransfer,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
_T = TypeVar("_T")
@dataclass
class MemoryAllocation:
"""Describe a tensor view carved from the Consumer receive slab."""
offset: int
size: int
tensor: torch.Tensor
@dataclass
class _ResidentEntry(Generic[_T]):
"""Store one resident value and its ownership accounting."""
value: _T
pinned: bool = True
leases: int = 0
@dataclass
class ResidentLease(Generic[_T]):
"""Represent a borrow of one resident entry."""
key: str
_entry: _ResidentEntry[_T]
_active: bool = True
@property
def value(self) -> _T:
return self._entry.value
class ContiguousAllocator:
"""Own a registered slab and its aligned regions; callers serialize access."""
def __init__(self, capacity: int, alignment: int = 256):
self.alignment = alignment
self._capacity = capacity
self._free = [(0, capacity)]
self.tensor: torch.Tensor | None = None
self._disabled = False
def prepare(self, device: torch.device, transfer: MooncakeTransfer) -> None:
if self.tensor is not None or self._disabled:
return
try:
tensor = torch.empty(self._capacity, dtype=torch.uint8, device=device)
ret = transfer.register_memory(tensor)
if ret != 0:
raise RuntimeError(f"Mooncake returned {ret}")
except (RuntimeError, torch.OutOfMemoryError) as error:
self._disabled = True
logger.warning("Could not initialize the EC registered buffer: %s", error)
return
self.tensor = tensor
self._free = [(0, tensor.nbytes)]
logger.info("Registered %d-byte buffer for Mooncake EC", tensor.nbytes)
def view(
self, offset: int, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> torch.Tensor:
assert self.tensor is not None
return self.tensor.narrow(0, offset, nbytes).view(dtype).view(shape)
def close(self, transfer: MooncakeTransfer) -> bool:
if self.tensor is not None and not transfer.unregister_memory(self.tensor):
return False
self.tensor = None
self._free.clear()
return True
def allocate(self, nbytes: int) -> tuple[int, int] | None:
size = (nbytes + self.alignment - 1) // self.alignment * self.alignment
for index, (offset, available) in enumerate(self._free):
if size > available:
continue
if size == available:
self._free.pop(index)
else:
self._free[index] = (offset + size, available - size)
return offset, size
return None
def free(self, offset: int, size: int) -> None:
index = bisect.bisect_left(self._free, (offset, size))
self._free.insert(index, (offset, size))
if index + 1 < len(self._free):
next_offset, next_size = self._free[index + 1]
if offset + size == next_offset:
self._free[index] = (offset, size + next_size)
self._free.pop(index + 1)
if index > 0:
previous_offset, previous_size = self._free[index - 1]
current_offset, current_size = self._free[index]
if previous_offset + previous_size == current_offset:
self._free[index - 1] = (
previous_offset,
previous_size + current_size,
)
self._free.pop(index)
class ResidentPool(Generic[_T]):
"""Track resident values, active leases, and LRU eviction eligibility."""
def __init__(self):
self._entries: dict[str, _ResidentEntry[_T]] = {}
self._evictable: OrderedDict[str, None] = OrderedDict()
def referenced(self) -> list[str]:
return [key for key, entry in self._entries.items() if entry.pinned]
def get(self, key: str) -> _T | None:
entry = self._entries.get(key)
return entry.value if entry is not None else None
def insert(
self,
key: str,
value: _T,
) -> _T | None:
"""Pin an entry and return a displaced value that has no owners."""
previous = self._entries.get(key)
entry = _ResidentEntry(value)
self._entries[key] = entry
self._evictable.pop(key, None)
if previous is not None and previous.leases == 0:
return previous.value
return None
def pin(self, key: str) -> _T | None:
entry = self._entries.get(key)
if entry is None:
return None
self._evictable.pop(key, None)
entry.pinned = True
return entry.value
def acquire(self, key: str) -> ResidentLease[_T] | None:
entry = self._entries.get(key)
if entry is None:
return None
entry.leases += 1
self._evictable.pop(key, None)
return ResidentLease(key, entry)
def release(self, lease: ResidentLease[_T]) -> _T | None:
if not lease._active:
return None
lease._active = False
entry = lease._entry
entry.leases -= 1
current = self._entries.get(lease.key)
if current is not entry:
if entry.leases == 0:
return entry.value
return None
if not entry.pinned and entry.leases == 0:
self._evictable[lease.key] = None
return None
def consume(self, lease: ResidentLease[_T]) -> tuple[_T, _T | None]:
current = self._entries[lease.key]
current.pinned = True
self._evictable.pop(lease.key, None)
released = self.release(lease)
return current.value, released
def retire(self, key: str) -> None:
if key not in self._entries:
return
entry = self._entries[key]
entry.pinned = False
if entry.leases == 0:
self._evictable[key] = None
def evict_lru(self, evict: Callable[[str, _T], bool]) -> str | None:
for key in list(self._evictable):
entry = self._entries[key]
if not evict(key, entry.value):
continue
self._evictable.pop(key, None)
del self._entries[key]
return key
return None
def clear(self) -> None:
self._entries.clear()
self._evictable.clear()
@dataclass
class StagedSources:
"""Own Producer tensor views and their staging-slab regions."""
tensors: list[torch.Tensor]
regions: list[tuple[int, int]]
class ProducerMemoryPool:
"""Own the Producer staging slab and regions carved from it."""
def __init__(self, capacity: int, transfer: MooncakeTransfer) -> None:
self._transfer = transfer
self._allocator = ContiguousAllocator(capacity)
self._lock = threading.Lock()
self._local = threading.local()
@property
def tensor(self) -> torch.Tensor | None:
return self._allocator.tensor
def _free_regions(self, regions: list[tuple[int, int]]) -> None:
for offset, size in regions:
self._allocator.free(offset, size)
def stage(self, tensors: list[torch.Tensor]) -> StagedSources | None:
"""Copy tensors into one registered slab, or return None for fallback."""
if not tensors:
return StagedSources([], [])
allocator = self._allocator
staged: list[torch.Tensor] = []
regions: list[tuple[int, int]] = []
with self._lock:
allocator.prepare(tensors[0].device, self._transfer)
pool = allocator.tensor
if pool is None:
return None
for tensor in tensors:
region = allocator.allocate(tensor.nbytes)
if region is None:
self._free_regions(regions)
return None
regions.append(region)
staged.append(
allocator.view(
region[0], tensor.nbytes, tuple(tensor.shape), tensor.dtype
)
)
if pool.device.type == "cuda":
stream = getattr(self._local, "stream", None)
if stream is None:
stream = torch.cuda.Stream(device=pool.device)
self._local.stream = stream
with torch.cuda.stream(stream):
for destination, source in zip(staged, tensors):
destination.copy_(source, non_blocking=True)
stream.synchronize()
else:
for destination, source in zip(staged, tensors):
destination.copy_(source)
return StagedSources(staged, regions)
def release(self, staged: StagedSources) -> None:
if not staged.regions:
return
with self._lock:
self._free_regions(staged.regions)
def close(self) -> None:
with self._lock:
self._allocator.close(self._transfer)
class ConsumerMemoryPool:
"""Own the registered receive slab and resident allocation lifecycle."""
def __init__(
self,
capacity: int,
transfer: MooncakeTransfer,
) -> None:
self._transfer = transfer
self._allocator = ContiguousAllocator(capacity)
self._residents: ResidentPool[MemoryAllocation] = ResidentPool()
self._retire_events: dict[str, torch.Event] = {}
self._pending_frees: list[tuple[torch.Event, MemoryAllocation]] = []
self._reclaimed: set[str] = set()
self.lock = threading.RLock()
@property
def tensor(self) -> torch.Tensor | None:
return self._allocator.tensor
def prepare(
self,
device: torch.device,
) -> None:
with self.lock:
self._allocator.prepare(device, self._transfer)
def _free(self, allocation: MemoryAllocation) -> None:
self._allocator.free(allocation.offset, allocation.size)
def free(self, allocation: MemoryAllocation) -> None:
with self.lock:
self._free(allocation)
def _defer_or_free(
self, allocation: MemoryAllocation, event: torch.Event | None
) -> None:
if event is None or event.query():
self._free(allocation)
else:
self._pending_frees.append((event, allocation))
def _poll_frees_locked(self) -> None:
pending = []
for event, allocation in self._pending_frees:
if event.query():
self._free(allocation)
else:
pending.append((event, allocation))
self._pending_frees = pending
def _reclaim_locked(self, nbytes: int) -> tuple[int, int] | None:
def evict(mm_hash: str, allocation: MemoryAllocation) -> bool:
event = self._retire_events.pop(mm_hash, None)
self._defer_or_free(allocation, event)
self._reclaimed.add(mm_hash)
return True
while self._residents.evict_lru(evict) is not None:
# Eviction defers any free whose CUDA event is still pending, so
# those bytes reach the allocator only once the event is polled.
self._poll_frees_locked()
region = self._allocator.allocate(nbytes)
if region is not None:
return region
self._poll_frees_locked()
return self._allocator.allocate(nbytes)
def _make_allocation(
self,
region: tuple[int, int],
nbytes: int,
shape: tuple[int, ...],
dtype: torch.dtype,
) -> MemoryAllocation:
offset, size = region
tensor = self._allocator.view(offset, nbytes, shape, dtype)
return MemoryAllocation(offset, size, tensor)
def try_allocate(
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> MemoryAllocation | None:
with self.lock:
allocator = self._allocator
assert allocator.tensor is not None
self._poll_frees_locked()
region = allocator.allocate(nbytes)
if region is None:
return None
return self._make_allocation(region, nbytes, shape, dtype)
def reclaim_and_allocate(
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> MemoryAllocation | None:
with self.lock:
region = self._reclaim_locked(nbytes)
if region is None:
return None
return self._make_allocation(region, nbytes, shape, dtype)
def acquire_cached(
self, mm_hash: str, shape: tuple[int, ...], dtype: torch.dtype
) -> ResidentLease[MemoryAllocation] | None:
with self.lock:
allocation = self._residents.get(mm_hash)
if allocation is None:
return None
if (
tuple(allocation.tensor.shape) != shape
or allocation.tensor.dtype != dtype
):
raise ValueError("conflicting cached tensor for mm_hash")
return self._residents.acquire(mm_hash)
def take_resident(
self, mm_hash: str, shape: tuple[int, ...], dtype_name: str
) -> torch.Tensor | None:
with self.lock:
allocation = self._residents.get(mm_hash)
if allocation is None:
return None
tensor = allocation.tensor
if (
tuple(tensor.shape) != shape
or str(tensor.dtype).split(".")[-1] != dtype_name
):
return None
self._residents.pin(mm_hash)
self._retire_events.pop(mm_hash, None)
return tensor
def _record_release_event(self) -> torch.Event | None:
pool = self.tensor
if pool is None or pool.device.type != "cuda":
return None
event = torch.Event()
event.record(torch.accelerator.current_stream(pool.device))
return event
def release_cached(self, lease: ResidentLease[MemoryAllocation]) -> None:
with self.lock:
released = self._residents.release(lease)
if released is not None:
self._defer_or_free(released, self._record_release_event())
def publish(
self,
mm_hash: str,
allocation: MemoryAllocation,
lease: ResidentLease[MemoryAllocation] | None = None,
*,
pin: bool = True,
) -> MemoryAllocation:
with self.lock:
if lease is not None:
canonical, released = self._residents.consume(lease)
self._retire_events.pop(mm_hash, None)
if released is not None:
self._defer_or_free(released, self._record_release_event())
return canonical
previous = self._residents.get(mm_hash)
displaced = self._residents.insert(mm_hash, allocation)
if not pin:
self._residents.retire(mm_hash)
event = None
if previous is not None and previous is not allocation:
event = self._retire_events.pop(mm_hash, None)
if displaced is None or displaced is allocation:
return allocation
if event is None:
event = self._record_release_event()
self._defer_or_free(displaced, event)
return allocation
def retire_stale(
self,
encoder_cache: dict[str, torch.Tensor],
reserved_hashes: set[str] | None = None,
*,
freed: list[str] | None = None,
) -> None:
if self.tensor is None:
return
with self.lock:
for mm_hash in self._residents.referenced() if freed is None else freed:
allocation = self._residents.get(mm_hash)
if allocation is None:
continue
if encoder_cache.get(mm_hash) is allocation.tensor:
continue
if reserved_hashes and mm_hash in reserved_hashes:
continue
event = self._record_release_event()
if event is not None:
self._retire_events[mm_hash] = event
self._residents.retire(mm_hash)
self._poll_frees_locked()
def drain_reclaimed(self) -> set[str]:
with self.lock:
reclaimed = self._reclaimed
self._reclaimed = set()
return reclaimed
def close(self) -> None:
with self.lock:
if not self._allocator.close(self._transfer):
return
self._residents.clear()
self._retire_events.clear()
self._pending_frees.clear()
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Metadata exchanged by the Mooncake encoder-cache connector."""
from __future__ import annotations
from dataclasses import dataclass, field
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
ECConnectorWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
TensorSpec,
)
@dataclass
class ECMooncakeLoadSpec:
"""Describe a remote reservation or resident allocation to load."""
mm_hash: str
nbytes: int
shape: tuple[int, ...]
dtype: str
transfer_id: str
# The consumer pool still holds this item, so the load is a local handoff:
# no transfer, no producer.
local: bool = False
@dataclass
class ECMooncakePushSpec:
"""Describe a destination reservation prepared before a tensor is ready."""
mm_hash: str
nbytes: int
shape: tuple[int, ...]
dtype: str
consumer_zmq: str
transfer_id: str
request_id: str = ""
@dataclass
class ECMooncakeConnectorMetadata(ECConnectorMetadata):
"""Worker operations emitted for one Scheduler step."""
loads: list[ECMooncakeLoadSpec] = field(default_factory=list)
pushes: list[ECMooncakePushSpec] = field(default_factory=list)
freed: list[str] | None = None
store_candidates: dict[str, TensorSpec] = field(default_factory=dict)
@dataclass
class ECMooncakeWorkerMetadata(ECConnectorWorkerMetadata):
"""Completion state reported from Workers to the Scheduler."""
loaded: set[str] = field(default_factory=set)
failed_loads: set[str] = field(default_factory=set)
# Items the receive pool dropped under pressure. The scheduler assumes an
# evicted item stays resident until told otherwise.
reclaimed: set[str] = field(default_factory=set)
pending_saves: bool = False
failed_saves: set[str] = field(default_factory=set)
def aggregate(self, other: ECConnectorWorkerMetadata) -> ECMooncakeWorkerMetadata:
assert isinstance(other, ECMooncakeWorkerMetadata)
return ECMooncakeWorkerMetadata(
# Every tensor-parallel rank gathers the embedding from its own
# cache, so an item counts as loaded only where all of them have
# it; one rank falling short must fail the load rather than leave
# the scheduler believing it is ready.
loaded=self.loaded & other.loaded,
failed_loads=self.failed_loads | other.failed_loads,
reclaimed=self.reclaimed | other.reclaimed,
pending_saves=self.pending_saves or other.pending_saves,
failed_saves=self.failed_saves | other.failed_saves,
)
@@ -1,418 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Producer-side push lifecycle, futures, and source-tensor ownership.
This module records when a push may advance or release its source. Worker
orchestration performs the actual control exchanges and Mooncake writes.
"""
from __future__ import annotations
import threading
from collections import OrderedDict
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import suppress
from dataclasses import dataclass, field
from enum import Enum, auto
from typing import Any
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakePushSpec,
)
def _same_destination(
existing: ECMooncakePushSpec, incoming: ECMooncakePushSpec
) -> bool:
"""Whether two specs describe one push, ignoring which request asked.
The same encoding requested twice shares a transfer id legitimately; only
a different payload or destination is a collision.
"""
return (
existing.mm_hash == incoming.mm_hash
and existing.nbytes == incoming.nbytes
and existing.shape == incoming.shape
and existing.dtype == incoming.dtype
and existing.consumer_zmq == incoming.consumer_zmq
)
class ProducerPushState(Enum):
"""Lifecycle of one Producer push from reservation to terminal state."""
# The reservation reply and source tensor may arrive in either order.
WAITING_INPUTS = auto()
WRITING = auto()
NOTIFYING = auto()
DONE = auto()
CANCEL_PENDING = auto()
CANCELLED = auto()
FAILED = auto()
@dataclass
class ProducerPushRecord:
"""Collect all asynchronous state for one Producer push."""
spec: ECMooncakePushSpec
state: ProducerPushState
reservation_future: Future[list[dict[str, Any]]]
reservations: list[dict[str, Any]] = field(default_factory=list)
shard_futures: list[Future[Any]] = field(default_factory=list)
source_tensor: torch.Tensor | None = None
source_event: torch.Event | None = None
batch_future: Future[None] | None = None
error: str | None = None
_ALLOWED_TRANSITIONS = {
ProducerPushState.WAITING_INPUTS: {
ProducerPushState.WRITING,
ProducerPushState.CANCEL_PENDING,
ProducerPushState.FAILED,
},
ProducerPushState.WRITING: {
ProducerPushState.NOTIFYING,
ProducerPushState.FAILED,
},
ProducerPushState.NOTIFYING: {
ProducerPushState.DONE,
ProducerPushState.FAILED,
},
ProducerPushState.CANCEL_PENDING: {
ProducerPushState.CANCELLED,
ProducerPushState.FAILED,
},
ProducerPushState.DONE: set(),
ProducerPushState.CANCELLED: set(),
ProducerPushState.FAILED: set(),
}
_TERMINAL_STATES = {
ProducerPushState.DONE,
ProducerPushState.CANCELLED,
ProducerPushState.FAILED,
}
_TERMINAL_LIMIT = 1 << 16
class ProducerPushManager:
"""Own Producer push records, transitions, and source tensor leases.
Terminal records are retained for duplicate-metadata idempotency. Separate
ordered indexes keep hot polling paths from scanning those tombstones.
"""
def __init__(self, wake: Callable[[], None] = lambda: None) -> None:
self._records: OrderedDict[str, ProducerPushRecord] = OrderedDict()
self._active_ids: OrderedDict[str, None] = OrderedDict()
self._reapable_terminal_ids: OrderedDict[str, None] = OrderedDict()
self._unreported_ids: OrderedDict[str, None] = OrderedDict()
self._batch_ids: OrderedDict[str, None] = OrderedDict()
self._source_waiters: dict[str, OrderedDict[str, None]] = {}
self._lock = threading.RLock()
self._wake = wake
def reserve(
self,
spec: ECMooncakePushSpec,
submit: Callable[[], Future[list[dict[str, Any]]]],
) -> tuple[ProducerPushRecord, bool]:
with self._lock:
existing = self._records.get(spec.transfer_id)
if existing is not None:
if not _same_destination(existing.spec, spec):
raise ValueError(
f"Conflicting EC destination for transfer_id={spec.transfer_id}"
)
return existing, False
record = ProducerPushRecord(
spec=spec,
state=ProducerPushState.WAITING_INPUTS,
reservation_future=submit(),
)
self._records[spec.transfer_id] = record
self._active_ids[spec.transfer_id] = None
self._source_waiters.setdefault(spec.mm_hash, OrderedDict())[
spec.transfer_id
] = None
record.reservation_future.add_done_callback(
lambda future: self._reservation_done(record, future)
)
return record, True
def bind_source(
self,
mm_hash: str,
tensor: torch.Tensor,
ready_event: torch.Event | None,
) -> None:
with self._lock:
waiters = self._source_waiters.pop(mm_hash, OrderedDict())
for transfer_id in waiters:
record = self._records[transfer_id]
if (
record.source_tensor is not None
or record.state is not ProducerPushState.WAITING_INPUTS
):
continue
record.source_tensor = tensor
record.source_event = ready_event
self._wake()
def submit_batches(
self,
executor: ThreadPoolExecutor,
run_batch: Callable[[list[ProducerPushRecord]], None],
*,
wait: bool = False,
) -> bool:
pending_event = False
with self._lock:
grouped: dict[str, list[ProducerPushRecord]] = {}
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if (
record.source_tensor is not None
and record.batch_future is None
and record.state is ProducerPushState.WAITING_INPUTS
):
if not record.reservation_future.done():
continue
if record.source_event is not None:
if wait:
record.source_event.synchronize()
elif not record.source_event.query():
pending_event = True
continue
grouped.setdefault(record.spec.consumer_zmq, []).append(record)
batches = list(grouped.values())
for records in batches:
future = executor.submit(run_batch, records)
for record in records:
record.batch_future = future
self._batch_ids[record.spec.transfer_id] = None
return pending_event
def resolve_reservations(self, record: ProducerPushRecord) -> list[dict[str, Any]]:
results = record.reservation_future.result()
with self._lock:
record.reservations = results
return list(results)
def finish_reservations(
self,
outcomes: list[tuple[ProducerPushRecord, list[dict[str, Any]] | Exception]],
) -> None:
with self._lock:
for record, result in outcomes:
if isinstance(result, Exception):
record.reservation_future.set_exception(result)
else:
record.reservation_future.set_result(result)
def _reservation_done(
self,
record: ProducerPushRecord,
future: Future[list[dict[str, Any]]],
) -> None:
try:
future.result()
except Exception as exc:
with self._lock:
self._set_error(record, exc)
if (
record.state is ProducerPushState.WAITING_INPUTS
and record.source_tensor is None
):
self._transition(record, ProducerPushState.FAILED)
finally:
self._wake()
def settle_all(self, records: list[ProducerPushRecord]) -> None:
for record in records:
with suppress(Exception):
record.reservation_future.result()
def replace_reservations(
self,
record: ProducerPushRecord,
reservations: list[dict[str, Any]],
) -> None:
with self._lock:
record.reservations = reservations
def track_shard_futures(
self,
records: list[ProducerPushRecord],
futures: list[Future[Any]],
) -> None:
with self._lock:
for record in records:
record.shard_futures.extend(futures)
def begin_writing(self, record: ProducerPushRecord) -> None:
with self._lock:
if record.source_tensor is None:
raise RuntimeError(
f"Producer push {record.spec.transfer_id!r} has no source tensor"
)
self._transition(record, ProducerPushState.WRITING)
def begin_notifying(self, records: list[ProducerPushRecord]) -> None:
with self._lock:
for record in records:
self._transition(record, ProducerPushState.NOTIFYING)
def complete(self, records: list[ProducerPushRecord]) -> None:
with self._lock:
self._check_sources_releasable(records)
for record in records:
self._transition(record, ProducerPushState.DONE)
self._release_source(record)
def fail(self, records: list[ProducerPushRecord], error: Exception) -> None:
with self._lock:
self._check_sources_releasable(records)
for record in records:
if record.state not in _TERMINAL_STATES:
self._set_error(record, error)
self._transition(record, ProducerPushState.FAILED)
self._release_source(record)
def cancel_requests(self, request_ids: set[str] | None) -> list[ProducerPushRecord]:
"""Cancel source-less waiters for requests, or all waiters at shutdown."""
cancelled = []
with self._lock:
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if (
(
request_ids is not None
and record.spec.request_id not in request_ids
)
or record.source_tensor is not None
or record.batch_future is not None
):
continue
if record.state is not ProducerPushState.WAITING_INPUTS:
continue
self._transition(record, ProducerPushState.CANCEL_PENDING)
cancelled.append(record)
return cancelled
def finish_cancel(self, record: ProducerPushRecord) -> None:
with self._lock:
if record.state is ProducerPushState.CANCEL_PENDING:
self._transition(record, ProducerPushState.CANCELLED)
def submit_cancel(
self,
record: ProducerPushRecord,
executor: ThreadPoolExecutor,
run_cancel: Callable[[ProducerPushRecord], None],
) -> None:
with self._lock:
future = executor.submit(run_cancel, record)
record.batch_future = future
transfer_id = record.spec.transfer_id
self._batch_ids[transfer_id] = None
if record.state in _TERMINAL_STATES:
self._reapable_terminal_ids.pop(transfer_id, None)
def poll(self) -> list[tuple[str, str]]:
failures = []
with self._lock:
for transfer_id in list(self._batch_ids):
record = self._records[transfer_id]
future = record.batch_future
assert future is not None
if not future.done():
continue
try:
future.result()
except Exception as exc:
self._set_error(record, exc)
self._batch_ids.pop(transfer_id)
self._mark_reapable(record)
for transfer_id in list(self._unreported_ids):
if transfer_id in self._batch_ids:
continue
record = self._records[transfer_id]
assert record.error is not None
failures.append((record.spec.mm_hash, record.error))
self._unreported_ids.pop(transfer_id)
self._mark_reapable(record)
self._reap_terminals()
return failures
@property
def pending(self) -> bool:
with self._lock:
return bool(self._active_ids or self._batch_ids)
def _release_source(self, record: ProducerPushRecord) -> None:
record.source_tensor = None
record.source_event = None
def _set_error(self, record: ProducerPushRecord, error: BaseException) -> None:
if record.error is None:
record.error = str(error)
if record.state in _TERMINAL_STATES:
transfer_id = record.spec.transfer_id
self._unreported_ids[transfer_id] = None
self._reapable_terminal_ids.pop(transfer_id, None)
def _reap_terminals(self) -> None:
while len(self._reapable_terminal_ids) > _TERMINAL_LIMIT:
transfer_id, _ = self._reapable_terminal_ids.popitem(last=False)
self._records.pop(transfer_id)
def _mark_reapable(self, record: ProducerPushRecord) -> None:
transfer_id = record.spec.transfer_id
if (
record.state in _TERMINAL_STATES
and transfer_id not in self._batch_ids
and transfer_id not in self._unreported_ids
):
self._reapable_terminal_ids[transfer_id] = None
def _drop_source_waiter(self, record: ProducerPushRecord) -> None:
waiters = self._source_waiters.get(record.spec.mm_hash)
if waiters is None:
return
waiters.pop(record.spec.transfer_id, None)
if not waiters:
self._source_waiters.pop(record.spec.mm_hash)
@staticmethod
def _check_sources_releasable(records: list[ProducerPushRecord]) -> None:
for record in records:
futures = [record.reservation_future, *record.shard_futures]
if record.source_tensor is not None and not all(
future.done() for future in futures
):
raise RuntimeError(
f"Producer push {record.spec.transfer_id!r} released its "
"source too early"
)
def _transition(self, record: ProducerPushRecord, state: ProducerPushState) -> None:
if state not in _ALLOWED_TRANSITIONS[record.state]:
raise RuntimeError(
f"Cannot transition producer push {record.spec.transfer_id!r} from "
f"{record.state.name} to {state.name}"
)
record.state = state
if state is not ProducerPushState.WAITING_INPUTS:
self._drop_source_waiter(record)
if state in _TERMINAL_STATES:
transfer_id = record.spec.transfer_id
self._active_ids.pop(transfer_id, None)
if record.error is not None:
self._unreported_ids[transfer_id] = None
self._mark_reapable(record)
@@ -1,487 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Consumer-side reservation lifecycle and destination-memory ownership.
Reservations make remote writes idempotent and ensure cancellation or expiry
cannot free a destination while Mooncake may still be writing into it.
"""
from __future__ import annotations
import threading
import time
import uuid
from collections import OrderedDict
from dataclasses import dataclass
from enum import Enum, auto
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
ConsumerMemoryPool,
MemoryAllocation,
ResidentLease,
)
class ConsumerReservationState(Enum):
"""Lifecycle of one Consumer destination reservation."""
WRITING = auto()
READY = auto()
CANCEL_PENDING = auto()
EXPIRE_PENDING = auto()
CANCELLED = auto()
EXPIRED = auto()
_ALLOWED_TRANSITIONS = {
ConsumerReservationState.WRITING: {
ConsumerReservationState.READY,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
ConsumerReservationState.CANCELLED,
},
ConsumerReservationState.READY: {
ConsumerReservationState.CANCELLED,
ConsumerReservationState.EXPIRED,
},
ConsumerReservationState.CANCEL_PENDING: {ConsumerReservationState.CANCELLED},
ConsumerReservationState.EXPIRE_PENDING: {
ConsumerReservationState.CANCELLED,
ConsumerReservationState.EXPIRED,
},
ConsumerReservationState.CANCELLED: set(),
ConsumerReservationState.EXPIRED: set(),
}
_ACTIVE_STATES = {
ConsumerReservationState.WRITING,
ConsumerReservationState.READY,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
_DEFERRED_STATES = {
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
_WRITER_OWNED_STATES = {
ConsumerReservationState.WRITING,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
@dataclass
class ConsumerReservation:
"""Own the identity and destination allocation of one remote push."""
transfer_id: str
mm_hash: str
reservation_id: str
state: ConsumerReservationState
shape: tuple[int, ...] = ()
dtype: str = ""
allocation: MemoryAllocation | None = None
lease: ResidentLease[MemoryAllocation] | None = None
expires_at: float = 0
writer_id: str = ""
class ConsumerReservationManager:
"""Own reservation transitions and destination allocation releases.
A remote writer owns a ``WRITING`` allocation. Cancellation and expiry are
therefore deferred until completion, unless refresh explicitly abandons
that writer before replacing its reservation.
"""
def __init__(
self,
memory: ConsumerMemoryPool,
lease_ttl: float,
tombstone_limit: int,
) -> None:
self._memory = memory
self._lease_ttl = lease_ttl
self._tombstone_limit = tombstone_limit
self._records: dict[str, ConsumerReservation] = {}
self._active_ids: dict[str, None] = {}
self._tombstones: OrderedDict[str, None] = OrderedDict()
self._writers: dict[str, ConsumerReservation] = {}
self._followers: dict[str, dict[str, None]] = {}
self._ready: list[str] = []
self._condition = threading.Condition(memory.lock)
self._shutting_down = False
def reserve(
self,
transfer_id: str,
mm_hash: str,
nbytes: int,
shape: tuple[int, ...],
dtype_name: str,
dtype: torch.dtype,
) -> tuple[ConsumerReservation | None, bool]:
with self._condition:
if self._shutting_down:
raise RuntimeError("Consumer reservation manager is shutting down")
existing = self._records.get(transfer_id)
if (
existing is not None
and existing.state is ConsumerReservationState.CANCELLED
):
return existing, False
if (
existing is not None
and existing.state is ConsumerReservationState.EXPIRED
):
self._remove(transfer_id)
existing = None
if (
existing is not None
and existing.state is ConsumerReservationState.EXPIRE_PENDING
):
raise RuntimeError(
f"Transfer {transfer_id!r} still has an active writer"
)
if existing is not None:
if (
existing.mm_hash != mm_hash
or existing.shape != shape
or existing.dtype != dtype_name
):
raise ValueError("conflicting reservation for transfer_id")
if existing.state not in _ACTIVE_STATES:
raise RuntimeError(
f"Cannot reserve transfer {transfer_id!r} in "
f"{existing.state.name}"
)
if existing.state is ConsumerReservationState.WRITING:
existing.expires_at = time.monotonic() + self._lease_ttl
return existing, False
lease = self._memory.acquire_cached(mm_hash, shape, dtype)
now = time.monotonic()
if lease is not None:
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.READY,
shape,
dtype_name,
lease.value,
lease,
now + self._lease_ttl,
)
self._insert(record)
return record, False
writer = self._writers.get(mm_hash)
if writer is not None and writer.state is ConsumerReservationState.WRITING:
if writer.shape != shape or writer.dtype != dtype_name:
raise ValueError("conflicting in-flight tensor for mm_hash")
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.WRITING,
shape,
dtype_name,
writer.allocation,
expires_at=now + self._lease_ttl,
writer_id=writer.transfer_id,
)
self._insert(record)
self._followers.setdefault(writer.transfer_id, {})[transfer_id] = None
return record, False
allocation = self._memory.try_allocate(nbytes, shape, dtype)
if allocation is None:
self._expire_locked(time.monotonic())
allocation = self._memory.try_allocate(nbytes, shape, dtype)
if allocation is None:
allocation = self._memory.reclaim_and_allocate(nbytes, shape, dtype)
if allocation is None:
return None, False
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.WRITING,
shape,
dtype_name,
allocation,
None,
now + self._lease_ttl,
)
self._insert(record)
self._writers[mm_hash] = record
return record, True
def status(self, transfer_id: str) -> ConsumerReservation | None:
with self._memory.lock:
record = self._records.get(transfer_id)
if record is None or record.state not in _ACTIVE_STATES:
return None
return record
def complete(self, transfer_id: str, reservation_id: str) -> tuple[bool, bool]:
with self._condition:
try:
record = self._records.get(transfer_id)
if record is None or record.reservation_id != reservation_id:
return False, False
if record.state is ConsumerReservationState.READY:
return True, False
if record.writer_id:
return False, False
if record.state in _WRITER_OWNED_STATES:
self._publish_followers(record)
if record.state in _DEFERRED_STATES:
terminal = (
ConsumerReservationState.CANCELLED
if record.state is ConsumerReservationState.CANCEL_PENDING
else ConsumerReservationState.EXPIRED
)
self._terminate(record, terminal)
return True, False
if record.state is not ConsumerReservationState.WRITING:
return False, False
self._transition(record, ConsumerReservationState.READY)
record.expires_at = time.monotonic() + self._lease_ttl
return True, True
finally:
self._condition.notify_all()
def _publish_followers(self, writer: ConsumerReservation) -> None:
if self._writers.get(writer.mm_hash) is writer:
self._writers.pop(writer.mm_hash)
followers = self._followers.pop(writer.transfer_id, {})
if not followers:
return
assert writer.allocation is not None
allocation = self._memory.publish(writer.mm_hash, writer.allocation, pin=False)
for record in [writer, *(self._records[key] for key in followers)]:
record.allocation = allocation
record.lease = self._memory.acquire_cached(
record.mm_hash, record.shape, allocation.tensor.dtype
)
assert record.lease is not None
if record is not writer:
record.writer_id = ""
self._transition(record, ConsumerReservationState.READY)
record.expires_at = time.monotonic() + self._lease_ttl
self._ready.append(record.transfer_id)
def drain_ready(self) -> list[str]:
with self._memory.lock:
ready, self._ready = self._ready, []
return ready
def begin_shutdown(self) -> None:
"""Stop new reservations and cancel everything without a remote writer."""
with self._condition:
if self._shutting_down:
return
self._shutting_down = True
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.writer_id or record.state is ConsumerReservationState.READY:
self._terminate(record, ConsumerReservationState.CANCELLED)
elif record.state is ConsumerReservationState.WRITING:
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
self._condition.notify_all()
def wait_for_writers(self, timeout: float) -> bool:
"""Wait until no reservation is still owned by a remote writer."""
def writers_finished() -> bool:
return not any(
self._records[transfer_id].state in _WRITER_OWNED_STATES
for transfer_id in self._active_ids
)
with self._condition:
return self._condition.wait_for(writers_finished, timeout=max(0, timeout))
def cancel(
self,
transfer_id: str,
reservation_id: str,
abandon: bool = False,
refresh: bool = False,
) -> bool:
with self._memory.lock:
record = self._records.get(transfer_id)
if (
record is not None
and reservation_id
and record.reservation_id != reservation_id
):
return False
if record is None:
record = ConsumerReservation(
transfer_id,
"",
"",
ConsumerReservationState.CANCELLED,
)
self._insert(record)
self._set_tombstone_deadline(record)
self._reap_tombstones(time.monotonic())
return True
if record.state is ConsumerReservationState.CANCELLED:
self._set_tombstone_deadline(record)
self._reap_tombstones(time.monotonic())
return True
if refresh:
if not abandon or record.state not in {
ConsumerReservationState.WRITING,
ConsumerReservationState.EXPIRE_PENDING,
}:
return False
if record.state is ConsumerReservationState.WRITING:
self._transition(record, ConsumerReservationState.EXPIRE_PENDING)
self._terminate(record, ConsumerReservationState.EXPIRED)
self._condition.notify_all()
self._reap_tombstones(time.monotonic())
return True
if record.writer_id:
self._terminate(record, ConsumerReservationState.CANCELLED)
return True
if record.state in _DEFERRED_STATES and not abandon:
return True
if record.state is ConsumerReservationState.WRITING and not abandon:
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
return True
if record.state not in _ACTIVE_STATES:
return False
self._terminate(record, ConsumerReservationState.CANCELLED)
if self._shutting_down:
self._condition.notify_all()
self._reap_tombstones(time.monotonic())
return True
def take(self, transfer_id: str, mm_hash: str) -> MemoryAllocation:
with self._memory.lock:
record = self._records.get(transfer_id)
if (
record is None
or record.state is not ConsumerReservationState.READY
or record.mm_hash != mm_hash
or record.allocation is None
):
raise RuntimeError(
f"Pushed EC tensor is not ready for mm_hash={mm_hash}"
)
allocation = self._memory.publish(mm_hash, record.allocation, record.lease)
record.allocation = None
record.lease = None
self._remove(transfer_id)
return allocation
def expire(self) -> int:
with self._memory.lock:
return self._expire_locked(time.monotonic())
def retire_stale(
self, encoder_cache: dict[str, torch.Tensor], freed: list[str] | None = None
) -> None:
with self._memory.lock:
self._memory.retire_stale(encoder_cache, freed=freed)
def _expire_locked(self, now: float) -> int:
expired = 0
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.expires_at > now:
continue
if record.state is ConsumerReservationState.READY:
self._terminate(record, ConsumerReservationState.EXPIRED)
expired += 1
elif record.state is ConsumerReservationState.WRITING:
if record.writer_id:
self._terminate(record, ConsumerReservationState.CANCELLED)
else:
self._defer(record, ConsumerReservationState.EXPIRE_PENDING)
self._reap_tombstones(now)
return expired
def _defer(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
"""Keep the destination until the writer completes or abandons it."""
self._transition(record, state)
record.expires_at = float("inf")
def _terminate(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
self._transition(record, state)
self._release(record)
self._set_tombstone_deadline(record)
def _release(self, record: ConsumerReservation) -> None:
if record.writer_id:
followers = self._followers.get(record.writer_id, {})
followers.pop(record.transfer_id, None)
record.allocation = None
return
if self._writers.get(record.mm_hash) is record:
self._writers.pop(record.mm_hash)
for transfer_id in list(self._followers.pop(record.transfer_id, {})):
self._terminate(
self._records[transfer_id], ConsumerReservationState.CANCELLED
)
allocation = record.allocation
if allocation is None:
return
if record.lease is not None:
self._memory.release_cached(record.lease)
else:
self._memory.free(allocation)
record.allocation = None
record.lease = None
def _set_tombstone_deadline(self, record: ConsumerReservation) -> None:
record.expires_at = time.monotonic() + self._lease_ttl
self._tombstones[record.transfer_id] = None
self._tombstones.move_to_end(record.transfer_id)
def _reap_tombstones(self, now: float) -> int:
dropped = 0
while self._tombstones:
transfer_id = next(iter(self._tombstones))
record = self._records[transfer_id]
if (
record.expires_at > now
and len(self._tombstones) <= self._tombstone_limit
):
break
self._remove(transfer_id)
dropped += 1
return dropped
def _insert(self, record: ConsumerReservation) -> None:
self._records[record.transfer_id] = record
if record.state in _ACTIVE_STATES:
self._active_ids[record.transfer_id] = None
def _remove(self, transfer_id: str) -> None:
self._records.pop(transfer_id, None)
self._active_ids.pop(transfer_id, None)
self._tombstones.pop(transfer_id, None)
def _transition(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
if state not in _ALLOWED_TRANSITIONS[record.state]:
raise RuntimeError(
f"Cannot transition {record.transfer_id!r} from "
f"{record.state.name} to {state.name}"
)
record.state = state
if state in _ACTIVE_STATES:
self._active_ids[record.transfer_id] = None
else:
self._active_ids.pop(record.transfer_id, None)
@@ -1,561 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-side planning and observation for Mooncake cache transfers.
The Scheduler prepares Producer reservations, consumes Consumer readiness
events, emits per-step Worker metadata, and converts Worker results back into
request availability without owning tensor memory or running data transfers.
"""
from __future__ import annotations
import math
import time
from collections import OrderedDict
from concurrent.futures import Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
get_encoder_output_width,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
_RESERVATION_TTL_SECONDS,
MooncakeECConfig,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
ControlClient,
EventInbox,
make_cancel_request,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeConnectorMetadata,
ECMooncakeLoadSpec,
ECMooncakePushSpec,
ECMooncakeWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.state import (
SchedulerTransferState,
SchedulerTransferTable,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
ensure_mooncake_available,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
TensorSpec,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
# Unmatched `ec_items` entries are a protocol desync, not a per-request
# event: warn on the first few, then only once in a while.
_MAX_UNRESOLVED_TRANSFER_ID_WARNINGS = 5
_DRAIN_MIN_INTERVAL = 0.005
_MAX_PENDING_EVENTS = 4096
_MAX_TERMINAL_TRANSFER_RECORDS = 1 << 16
_CANCEL_ATTEMPTS = 2
_CONTROL_WORKERS = 8
class ECMooncakeScheduler:
"""Coordinate Mooncake transfers from the vLLM Scheduler process."""
def __init__(self, vllm_config: VllmConfig) -> None:
ensure_mooncake_available()
config = MooncakeECConfig.from_vllm_config(vllm_config)
self._store_candidates: list[tuple[str, str, TensorSpec]] = []
self._cross_encoder_cache = config.cross_encoder_cache
self._is_producer = config.is_producer
self._is_consumer = config.is_consumer
self._control_addr = config.control_addr
self._push_wait_timeout = config.push_wait_timeout_s
self._vllm_config = vllm_config
self._model_config = vllm_config.model_config
self._control_client = ControlClient(config.control_timeout_ms)
self._control_executor = ThreadPoolExecutor(
max_workers=_CONTROL_WORKERS,
thread_name_prefix="ec-mooncake-control",
)
self._event_inbox = EventInbox(self._control_client, self._control_executor)
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
self._unresolved_transfer_ids = 0
self._drain_pending = True
self._drained_at = 0.0
self._pending_cancels: dict[str, Future[Any]] = {}
self._transfers = SchedulerTransferTable(
config.pool_size, _RESERVATION_TTL_SECONDS
)
self._scheduler_pending_work = False
self._pushes_to_prepare: dict[str, ECMooncakePushSpec] = {}
self._prepared_push_transfer_ids: set[str] = set()
self._event_ready_shards: OrderedDict[str, set[int]] = OrderedDict()
# Mirror of the engine's encoder cache, maintained from the alloc and
# free notifications the Scheduler already sends. Tracking it here
# keeps `ensure_cache_available` on the upstream two-argument shape.
self._local_cache: set[str] = set()
self._failed_saves: set[str] = set()
def _cancel_remote(self, consumer_zmq: str, transfer_id: str) -> bool:
pending = None
for _ in range(_CANCEL_ATTEMPTS):
pending = self._control_client.discover_shards(consumer_zmq)
if pending is not None:
break
if pending is None:
raise RuntimeError(
f"Could not discover every EC consumer shard at {consumer_zmq}"
)
cancelled = False
error: BaseException | None = None
for _ in range(_CANCEL_ATTEMPTS):
failed = []
for addr in pending:
try:
result = self._control_client.request(
addr,
make_cancel_request(transfer_id),
)
except Exception as exc:
if error is None:
error = exc
failed.append(addr)
continue
cancelled |= isinstance(result, dict) and bool(result.get("cancelled"))
if not failed:
return cancelled
pending = failed
if error is not None:
raise error
return cancelled
def _note_awaiting_push(
self,
mm_hash: str,
transfer_id: str,
request_id: str,
now: float,
) -> None:
record = self._transfers.wait_for_event(
transfer_id,
request_id,
mm_hash,
now + self._push_wait_timeout,
)
if record is None:
# The id names another encoding; the request is already failed.
return
if record.state is not SchedulerTransferState.WAITING_EVENT:
return
assert record.deadline is not None
if now < record.deadline:
return
elapsed = now - record.deadline + self._push_wait_timeout
self._transfers.mark_unavailable(transfer_id, now)
logger.warning(
"EC Mooncake waited %.1fs for a push of mm_hash=%s "
"(transfer_id=%s) that never arrived; "
"requests needing it fail with a retryable error.",
elapsed,
mm_hash,
transfer_id,
)
def take_unavailable_requests(self) -> set[str]:
failed, self._failed_saves = self._failed_saves, set()
return failed | self._transfers.take_unavailable_requests()
def _poll_pending_cancels(self) -> None:
pending = {}
for transfer_id, future in self._pending_cancels.items():
if not future.done():
pending[transfer_id] = future
continue
try:
future.result()
except Exception:
logger.warning(
"EC Mooncake reservation cancellation failed", exc_info=True
)
self._pending_cancels = pending
def _note_shard_ready(self, data: dict[str, Any]) -> bool:
shard_count = self._event_inbox.shard_count
if shard_count <= 1:
return True
transfer_id = str(data["transfer_id"])
record = self._transfers.get(transfer_id)
if (
record is not None
and record.state is not SchedulerTransferState.WAITING_EVENT
):
return False
shard = data.get("shard")
shards = self._event_ready_shards.setdefault(transfer_id, set())
self._event_ready_shards.move_to_end(transfer_id)
shards.add(int(shard) if shard is not None else len(shards))
if len(shards) < shard_count:
while len(self._event_ready_shards) > _MAX_PENDING_EVENTS:
self._event_ready_shards.popitem(last=False)
return False
self._event_ready_shards.pop(transfer_id, None)
return True
def _forget_shard_readiness(self, transfer_id: str) -> None:
self._event_ready_shards.pop(transfer_id, None)
def _store_pushed_spec(self, data: dict[str, Any]) -> bool:
transfer_id = str(data["transfer_id"])
identifier = str(data["mm_hash"])
_, accepted = self._transfers.observe_ready(
ECMooncakeLoadSpec(
mm_hash=identifier,
nbytes=int(data["nbytes"]),
shape=tuple(int(value) for value in data["shape"]),
dtype=str(data["dtype"]),
transfer_id=transfer_id,
),
time.monotonic() + _RESERVATION_TTL_SECONDS,
)
return accepted
def _queue_cancel(
self,
transfer_id: str,
mm_hash: str = "",
request_id: str = "",
) -> None:
if not self._transfers.cancel(
transfer_id,
time.monotonic(),
mm_hash=mm_hash,
request_id=request_id,
):
return
self._forget_shard_readiness(transfer_id)
self._pending_cancels[transfer_id] = self._control_executor.submit(
self._cancel_remote,
self._control_addr,
transfer_id,
)
def _expire_transfers(self) -> None:
now = time.monotonic()
expired = self._transfers.expire(now, _MAX_TERMINAL_TRANSFER_RECORDS)
for record in expired:
self._queue_cancel(record.transfer_id)
def _drain_push_notifications(self) -> None:
now = time.monotonic()
if not self._drain_pending and now - self._drained_at < _DRAIN_MIN_INTERVAL:
return
self._drain_pending = False
self._drained_at = now
self._poll_pending_cancels()
self._expire_transfers()
events = self._event_inbox.drain(self._control_addr)
for data in events:
try:
if not isinstance(data, dict):
raise TypeError("EC readiness event must be an object")
if not data.get("ready"):
continue
self._accept_ready_event(data)
except (KeyError, TypeError, ValueError):
# Readiness events cross a plain PULL socket, so a malformed
# one costs that event and not the engine.
logger.warning(
"Discarding a malformed EC readiness event.", exc_info=True
)
def _accept_ready_event(self, data: dict[str, Any]) -> None:
transfer_id = str(data["transfer_id"])
record = self._transfers.get(transfer_id)
if record is not None and record.state in {
SchedulerTransferState.CANCELLED,
SchedulerTransferState.UNAVAILABLE,
SchedulerTransferState.EXPIRED,
SchedulerTransferState.FAILED,
}:
return
if not self._note_shard_ready(data):
return
self._store_pushed_spec(data)
def has_cache_item(self, identifier: str) -> bool:
if not self._is_consumer:
return False
self._drain_push_notifications()
return self._transfers.has_state(
identifier,
(
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
SchedulerTransferState.AVAILABLE,
),
)
def _warn_unresolved_transfer_id(
self, request: Any, index: int, where: str
) -> None:
"""Report an `ec_items` entry that cannot be matched to a feature.
A caller that cannot resolve a transfer id has to fall back to a
locally invented one or skip its bookkeeping entirely, and either way
the two sides of a transfer stop agreeing on its name. That desyncs
silently -- reservations are never released and only surface minutes
later as a full buffer pool -- so say it out loud the first time.
`mm_hash` in `ec_items` must be a value some engine reported, never one
the caller derived itself: `mm_features[i].identifier` folds in the
engine's media_io_kwargs and mm_processor_kwargs, so a media uuid alone
no longer equals it. Omit the field to match by position instead.
"""
self._unresolved_transfer_ids += 1
count = self._unresolved_transfer_ids
if count > _MAX_UNRESOLVED_TRANSFER_ID_WARNINGS and count % 1000:
return
params = getattr(request, "ec_transfer_params", None)
items = (params or {}).get("ec_items") or []
logger.warning(
"EC Mooncake could not resolve a transfer id at %s for req=%s "
"index=%d: feature identifier=%s does not match any of the %d "
"ec_items %s (ec_transfer_params present=%s). Occurrence %d. "
"ec_items[].mm_hash must echo an engine-reported identifier, or "
"be omitted so the entry is matched by position.",
where,
request.request_id,
index,
request.mm_features[index].identifier[:16],
len(items),
[str(item.get("mm_hash"))[:16] for item in items[:4]],
params is not None,
count,
)
@staticmethod
def _request_transfer_id(request: Any, index: int) -> str | None:
params = getattr(request, "ec_transfer_params", None) or {}
items = params.get("ec_items") or []
mm_hash = request.mm_features[index].identifier
if index < len(items):
item = items[index]
if item.get("mm_hash") in (None, mm_hash) and item.get("transfer_id"):
return str(item["transfer_id"])
for item in items:
if item.get("mm_hash") == mm_hash and item.get("transfer_id"):
return str(item["transfer_id"])
return None
def ensure_cache_available(self, request: Any, num_computed_tokens: int) -> bool:
if self._is_producer:
for index, feature in enumerate(request.mm_features):
if (
feature.mm_position.offset + feature.mm_position.length
> num_computed_tokens
):
self._prepare_push_spec(request, index)
if not self._is_consumer:
return True
self._drain_push_notifications()
# One timestamp for the whole decision, so the deadlines it hands out
# cannot disagree between features.
now = time.monotonic()
all_ready = True
for index, feature in enumerate(request.mm_features):
if (
feature.mm_position.offset + feature.mm_position.length
<= num_computed_tokens
):
continue
mm_hash = feature.identifier
transfer_id = self._request_transfer_id(request, index)
if transfer_id is not None:
self._transfers.touch_available(
transfer_id, now + _RESERVATION_TTL_SECONDS
)
if mm_hash in self._local_cache:
continue
if self._transfers.has_state(mm_hash, (SchedulerTransferState.READY,)):
continue
if self._transfers.has_state(mm_hash, (SchedulerTransferState.LOADING,)):
all_ready = False
continue
record = self._transfers.begin_load(
mm_hash,
transfer_id,
request.request_id,
now + self._push_wait_timeout,
)
if record is not None:
self._scheduler_pending_work = True
all_ready = False
else:
waiting_id = transfer_id or f"{request.request_id}:{index}"
self._note_awaiting_push(mm_hash, waiting_id, request.request_id, now)
all_ready = False
return all_ready
def _prepare_push_spec(self, request: Any, index: int) -> None:
params = getattr(request, "ec_transfer_params", None) or {}
consumer_zmq = params.get("consumer_zmq")
mm_hash = request.mm_features[index].identifier
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
# Still push: after the proxy has rewritten the item to embeds the
# consumer has no media left to fall back on, so a nameless push
# beats none. But the consumer knows this transfer by the id it
# sent, not by the one invented here, so its cancel will never
# reach the reservation this push is about to take.
if consumer_zmq:
self._warn_unresolved_transfer_id(request, index, "push prepare")
transfer_id = f"{request.request_id}:{index}"
if not consumer_zmq or transfer_id in self._prepared_push_transfer_ids:
return
output = self._encoder_output_spec(request, index)
self._pushes_to_prepare[transfer_id] = ECMooncakePushSpec(
mm_hash=mm_hash,
nbytes=output.nbytes,
shape=output.shape,
dtype=output.dtype.removeprefix("torch."),
consumer_zmq=str(consumer_zmq),
transfer_id=transfer_id,
request_id=request.request_id,
)
self._prepared_push_transfer_ids.add(transfer_id)
def _encoder_output_spec(self, request: Any, index: int) -> TensorSpec:
dtype = self._model_config.dtype
assert isinstance(dtype, torch.dtype)
hidden_dim = get_encoder_output_width(
self._vllm_config, request.mm_features[index].modality
)
shape = (request.get_num_encoder_embeds(index), hidden_dim)
return TensorSpec(shape, str(dtype), math.prod(shape) * dtype.itemsize)
def update_state_after_alloc(self, request: Any, index: int) -> None:
self._local_cache.add(request.mm_features[index].identifier)
if self._is_producer:
self._prepare_push_spec(request, index)
if (
self._cross_encoder_cache
and request.mm_features[index].modality == "image"
):
self._store_candidates.append(
(
request.request_id,
request.mm_features[index].identifier,
self._encoder_output_spec(request, index),
)
)
def update_state_after_free(self, request: Any, index: int) -> None:
if not self._is_consumer:
return
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
self._warn_unresolved_transfer_id(request, index, "encoder-cache free")
return
self._queue_cancel(
transfer_id,
mm_hash=request.mm_features[index].identifier,
request_id=request.request_id,
)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> ECConnectorMetadata:
for mm_hash in scheduler_output.free_encoder_mm_hashes:
self._local_cache.discard(mm_hash)
self._transfers.release_ready(mm_hash, time.monotonic())
for transfer_id in self._transfers.drain_orphaned():
self._queue_cancel(transfer_id)
meta = ECMooncakeConnectorMetadata(
freed=scheduler_output.free_encoder_mm_hashes
)
preempted = scheduler_output.preempted_req_ids or set()
meta.store_candidates = {
identifier: output
for request_id, identifier, output in self._store_candidates
if request_id not in preempted
}
self._store_candidates.clear()
for push_spec in self._pushes_to_prepare.values():
meta.pushes.append(push_spec)
self._pushes_to_prepare.clear()
for record in self._transfers.take_loads_to_dispatch():
assert record.spec is not None
meta.loads.append(record.spec)
self._poll_pending_cancels()
self._drain_pending = True
return meta
def update_connector_output(self, connector_output: ECConnectorOutput) -> None:
meta = connector_output.ec_connector_worker_meta
if not isinstance(meta, ECMooncakeWorkerMetadata):
return
for mm_hash in meta.loaded:
self._transfers.complete_load(mm_hash)
for mm_hash in meta.failed_loads:
self._transfers.fail_load(mm_hash, time.monotonic())
for mm_hash in meta.reclaimed:
self._transfers.reclaim(mm_hash, time.monotonic())
self._scheduler_pending_work = meta.pending_saves
self._failed_saves.update(meta.failed_saves)
def has_pending_push_work(self) -> bool:
return self._scheduler_pending_work
def request_finished(self, request: Any) -> tuple[bool, dict[str, Any] | None]:
if self._is_consumer:
for index in range(len(request.mm_features)):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
self._warn_unresolved_transfer_id(request, index, "request finish")
continue
self._queue_cancel(
transfer_id,
mm_hash=request.mm_features[index].identifier,
request_id=request.request_id,
)
if self._is_producer and self._prepared_push_transfer_ids:
for index in range(len(request.mm_features)):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
transfer_id = f"{request.request_id}:{index}"
self._prepared_push_transfer_ids.discard(transfer_id)
if not self._is_producer:
return False, None
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
for index, feature in enumerate(request.mm_features):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is not None:
items[feature.identifier]["transfer_id"] = transfer_id
if not items:
return False, None
return False, items
def close(self) -> None:
self._control_executor.shutdown(wait=True, cancel_futures=True)
self._control_client.close()
self._event_inbox.close()
@@ -1,448 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-owned lifecycle and indexes for Consumer-bound transfers."""
from __future__ import annotations
from collections import OrderedDict, deque
from collections.abc import Iterable
from dataclasses import dataclass, field, replace
from enum import Enum, auto
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeLoadSpec,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
class SchedulerTransferState(Enum):
"""Lifecycle states visible to the Scheduler.
The states cover waiting for a Consumer event, dispatching a Worker load,
retaining a locally reusable tensor, and terminal failure conditions.
"""
WAITING_EVENT = auto()
AVAILABLE = auto()
LOADING = auto()
READY = auto()
RESIDENT = auto()
UNAVAILABLE = auto()
EXPIRED = auto()
FAILED = auto()
CANCELLED = auto()
@dataclass
class SchedulerTransfer:
"""Track one transfer as observed by the Scheduler."""
transfer_id: str
request_id: str
mm_hash: str
state: SchedulerTransferState
spec: ECMooncakeLoadSpec | None
deadline: float | None
notified_requests: set[str] = field(default_factory=set, repr=False)
_TERMINAL_STATES = {
SchedulerTransferState.UNAVAILABLE,
SchedulerTransferState.EXPIRED,
SchedulerTransferState.FAILED,
SchedulerTransferState.CANCELLED,
}
class SchedulerTransferTable:
"""Own Scheduler transfer state, lookup indexes, and dispatch queues.
The Scheduler is single-threaded, so direct transitions are sufficient;
the Producer and Consumer managers retain stricter transition matrices
because callbacks and control requests can race there.
"""
def __init__(self, resident_capacity: int, tombstone_ttl: float) -> None:
self._resident_capacity = resident_capacity
self._tombstone_ttl = tombstone_ttl
self._records: OrderedDict[str, SchedulerTransfer] = OrderedDict()
self._active_ids: dict[str, None] = {}
self._terminal_ids: OrderedDict[str, None] = OrderedDict()
self._resident_ids: OrderedDict[str, None] = OrderedDict()
self._resident_bytes = 0
self._hash_index: dict[str, deque[str]] = {}
self._loads_to_dispatch: OrderedDict[str, None] = OrderedDict()
self._unavailable_requests: set[str] = set()
# Records expired straight out of READY/RESIDENT: the consumer worker
# still holds their push reservation, and nothing else tells it to let
# go, so the destination buffer would sit pinned until the lease TTL.
self._orphaned: list[str] = []
def get(self, transfer_id: str) -> SchedulerTransfer | None:
return self._records.get(transfer_id)
def drain_orphaned(self) -> list[str]:
"""Transfer IDs expired out of READY/RESIDENT since the last drain.
The caller must release the consumer worker's reservation for each:
`cancel()` still accepts an EXPIRED record, so the usual cancel path
applies.
"""
orphaned = self._orphaned
self._orphaned = []
return orphaned
def records_for_hash(
self,
mm_hash: str,
states: Iterable[SchedulerTransferState],
) -> list[SchedulerTransfer]:
wanted = set(states)
return [
record
for transfer_id in self._hash_index.get(mm_hash, ())
if (record := self._records.get(transfer_id)) is not None
and record.state in wanted
]
def first_for_hash(
self,
mm_hash: str,
states: Iterable[SchedulerTransferState],
) -> SchedulerTransfer | None:
states = tuple(states)
if len(states) == 1:
wanted_state = states[0]
for transfer_id in self._hash_index.get(mm_hash, ()):
record = self._records.get(transfer_id)
if record is not None and record.state is wanted_state:
return record
return None
wanted_states = set(states)
for transfer_id in self._hash_index.get(mm_hash, ()):
record = self._records.get(transfer_id)
if record is not None and record.state in wanted_states:
return record
return None
def has_state(self, mm_hash: str, states: Iterable[SchedulerTransferState]) -> bool:
return self.first_for_hash(mm_hash, states) is not None
def wait_for_event(
self,
transfer_id: str,
request_id: str,
mm_hash: str,
deadline: float,
) -> SchedulerTransfer | None:
"""Track a request waiting for a push.
Returns None when `transfer_id` already names a different encoding.
The id comes from the request, so a collision has to fail that one
request; re-issuing it re-runs the encode under a fresh id.
"""
record = self._records.get(transfer_id)
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id=request_id,
mm_hash=mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=deadline,
)
self._insert(record)
elif not self._identity_matches(record, mm_hash):
self._refuse_colliding_id(record, mm_hash, request_id)
return None
else:
if not record.request_id:
record.request_id = request_id
if record.state in _TERMINAL_STATES and request_id:
self._notify_unavailable(record, request_id)
return record
def observe_ready(
self, spec: ECMooncakeLoadSpec, deadline: float
) -> tuple[SchedulerTransfer, bool]:
transfer_id = spec.transfer_id
record = self._records.get(transfer_id)
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id="",
mm_hash=spec.mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=None,
)
self._insert(record)
elif not self._identity_matches(record, spec.mm_hash):
self._refuse_colliding_id(record, spec.mm_hash, record.request_id)
return record, False
if record.state is not SchedulerTransferState.WAITING_EVENT:
return record, False
record.spec = spec
record.deadline = deadline
self._transition(record, SchedulerTransferState.AVAILABLE)
return record, True
def touch_available(self, transfer_id: str, deadline: float) -> None:
record = self._records.get(transfer_id)
if record is not None and record.state is SchedulerTransferState.AVAILABLE:
record.deadline = deadline
def begin_load(
self,
mm_hash: str,
transfer_id: str | None = None,
request_id: str = "",
deadline: float | None = None,
) -> SchedulerTransfer | None:
record = self._records.get(transfer_id) if transfer_id else None
if record is not None and (
record.mm_hash != mm_hash
or record.state is not SchedulerTransferState.AVAILABLE
):
record = None
if record is None:
record = self.first_for_hash(
mm_hash,
(
SchedulerTransferState.AVAILABLE,
SchedulerTransferState.RESIDENT,
),
)
if record is None or record.spec is None:
return None
if not record.request_id:
record.request_id = request_id
# A dispatched load that never reports back would otherwise hold this
# hash in LOADING for good, deferring every later request for it.
record.deadline = deadline
self._transition(record, SchedulerTransferState.LOADING)
self._loads_to_dispatch[record.transfer_id] = None
return record
def take_loads_to_dispatch(self) -> list[SchedulerTransfer]:
records = [
record
for transfer_id in self._loads_to_dispatch
if (record := self._records.get(transfer_id)) is not None
and record.state is SchedulerTransferState.LOADING
]
self._loads_to_dispatch.clear()
return records
def complete_load(self, mm_hash: str) -> bool:
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
if record is None:
return self.has_state(mm_hash, (SchedulerTransferState.READY,))
self._transition(record, SchedulerTransferState.READY)
return True
def fail_load(self, mm_hash: str, now: float) -> bool:
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
if record is None:
return self.has_state(mm_hash, (SchedulerTransferState.FAILED,))
self._transition(record, SchedulerTransferState.FAILED, now=now)
return True
def release_ready(self, mm_hash: str, now: float) -> None:
ready = self.records_for_hash(mm_hash, (SchedulerTransferState.READY,))
if ready:
canonical = ready[-1]
for record in self.records_for_hash(
mm_hash,
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
):
if record is not canonical:
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
if canonical.spec is None:
self._transition(canonical, SchedulerTransferState.EXPIRED, now=now)
else:
canonical.spec = replace(canonical.spec, local=True)
self._transition(canonical, SchedulerTransferState.RESIDENT)
self._evict_residents(now)
def reclaim(self, mm_hash: str, now: float) -> None:
for record in self.records_for_hash(
mm_hash,
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
):
if record.state is SchedulerTransferState.READY:
record.spec = None
else:
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
def mark_unavailable(self, transfer_id: str, now: float) -> None:
record = self._records[transfer_id]
self._transition(record, SchedulerTransferState.UNAVAILABLE, now=now)
if record.request_id:
self._notify_unavailable(record, record.request_id)
def cancel(
self,
transfer_id: str,
now: float,
mm_hash: str = "",
request_id: str = "",
) -> bool:
record = self._records.get(transfer_id)
if (
record is not None
and mm_hash
and not self._identity_matches(record, mm_hash)
):
return False
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id=request_id,
mm_hash=mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=None,
)
self._insert(record)
if record.state in {
SchedulerTransferState.CANCELLED,
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
SchedulerTransferState.FAILED,
}:
return False
self._transition(record, SchedulerTransferState.CANCELLED, now=now)
self._loads_to_dispatch.pop(transfer_id, None)
return True
def expire(self, now: float, terminal_limit: int) -> list[SchedulerTransfer]:
if terminal_limit < 0:
raise ValueError("terminal_limit must be non-negative")
expired = []
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.deadline is None or record.deadline > now:
continue
if record.state is SchedulerTransferState.AVAILABLE:
self._transition(
record,
SchedulerTransferState.EXPIRED,
now=now,
)
expired.append(record)
elif record.state is SchedulerTransferState.LOADING:
self._transition(
record,
SchedulerTransferState.UNAVAILABLE,
now=now,
)
if record.request_id:
self._notify_unavailable(record, record.request_id)
expired.append(record)
while self._terminal_ids:
transfer_id = next(iter(self._terminal_ids))
record = self._records[transfer_id]
if (
record.deadline is not None
and record.deadline > now
and len(self._terminal_ids) <= terminal_limit
):
break
self._remove(transfer_id)
return expired
def take_unavailable_requests(self) -> set[str]:
unavailable = self._unavailable_requests
self._unavailable_requests = set()
return unavailable
def _notify_unavailable(self, record: SchedulerTransfer, request_id: str) -> None:
if request_id not in record.notified_requests:
record.notified_requests.add(request_id)
self._unavailable_requests.add(request_id)
def _insert(self, record: SchedulerTransfer) -> None:
self._records[record.transfer_id] = record
self._active_ids[record.transfer_id] = None
if record.mm_hash:
self._hash_index.setdefault(record.mm_hash, deque()).append(
record.transfer_id
)
@staticmethod
def _identity_matches(record: SchedulerTransfer, mm_hash: str) -> bool:
return not record.mm_hash or record.mm_hash == mm_hash
def _refuse_colliding_id(
self, record: SchedulerTransfer, mm_hash: str, request_id: str
) -> None:
"""Report a transfer id that already names another encoding.
Transfer ids arrive on the request, so a collision is reachable from
outside and must not reach the engine as an exception.
"""
logger.warning(
"EC Mooncake transfer_id=%s already names mm_hash=%s; refusing "
"mm_hash=%s for request %s",
record.transfer_id,
record.mm_hash[:16],
mm_hash[:16],
request_id or "<unknown>",
)
if request_id:
self._unavailable_requests.add(request_id)
def _transition(
self,
record: SchedulerTransfer,
state: SchedulerTransferState,
now: float | None = None,
) -> None:
if record.state is SchedulerTransferState.RESIDENT:
self._resident_ids.pop(record.transfer_id, None)
if record.spec is not None:
self._resident_bytes -= record.spec.nbytes
if state is SchedulerTransferState.RESIDENT:
self._resident_ids[record.transfer_id] = None
if record.spec is not None:
self._resident_bytes += record.spec.nbytes
if state in _TERMINAL_STATES:
if now is None:
raise ValueError("Terminal transition requires a timestamp")
# Keep a bounded tombstone so late events and repeat request IDs
# remain idempotent instead of reviving a finished transfer.
record.deadline = now + self._tombstone_ttl
self._active_ids.pop(record.transfer_id, None)
self._terminal_ids[record.transfer_id] = None
self._terminal_ids.move_to_end(record.transfer_id)
if state is SchedulerTransferState.EXPIRED and record.state in (
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
):
self._orphaned.append(record.transfer_id)
record.state = state
self._records.move_to_end(record.transfer_id)
def _evict_residents(self, now: float) -> None:
while self._resident_bytes > self._resident_capacity and self._resident_ids:
record = self._records[next(iter(self._resident_ids))]
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
def _remove(self, transfer_id: str) -> None:
record = self._records.pop(transfer_id, None)
self._active_ids.pop(transfer_id, None)
self._terminal_ids.pop(transfer_id, None)
if transfer_id in self._resident_ids:
self._resident_ids.pop(transfer_id)
if record is not None and record.spec is not None:
self._resident_bytes -= record.spec.nbytes
self._loads_to_dispatch.pop(transfer_id, None)
if record is None or not record.mm_hash:
return
transfer_ids = self._hash_index[record.mm_hash]
transfer_ids.remove(transfer_id)
if not transfer_ids:
self._hash_index.pop(record.mm_hash)
@@ -1,209 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake data-plane engine and memory-registration ownership.
The wrapper isolates optional-engine initialization, session addressing,
synchronous batched writes, and the lifetime of transient source registrations.
"""
from __future__ import annotations
import threading
from dataclasses import dataclass
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
_MOONCAKE_IMPORT_ERROR: ImportError | None = None
try:
from mooncake.engine import TransferEngine
except ImportError as error:
TransferEngine = None # type: ignore[misc, assignment]
_MOONCAKE_IMPORT_ERROR = error
def ensure_mooncake_available() -> None:
if _MOONCAKE_IMPORT_ERROR is not None:
raise ImportError(
"Install mooncake-transfer-engine to use ECMooncakeConnector."
) from _MOONCAKE_IMPORT_ERROR
@dataclass
class _SourceRegistration:
"""Retain one transient source range while batches reference it."""
tensor: torch.Tensor
nbytes: int
users: int = 1
class MooncakeTransfer:
"""Own a lazy Mooncake engine and transient memory registrations."""
def __init__(self, hostname: str, protocol: str) -> None:
self._hostname = hostname
self._protocol = protocol
self._engine: TransferEngine | None = None
self._engine_lock = threading.Lock()
self._source_registrations: dict[int, _SourceRegistration] = {}
self._pending_unregister: dict[int, torch.Tensor] = {}
self._registration_lock = threading.Lock()
self._closed = False
def _ensure_engine(self) -> TransferEngine:
if self._engine is not None:
return self._engine
with self._engine_lock:
if self._engine is not None:
return self._engine
engine = TransferEngine()
ret = engine.initialize(self._hostname, "P2PHANDSHAKE", self._protocol, "")
if ret != 0:
raise RuntimeError("Mooncake TransferEngine initialization failed.")
self._engine = engine
logger.info(
"ECMooncakeConnector TransferEngine ready at %s:%d",
self._hostname,
engine.get_rpc_port(),
)
return self._engine
def ensure_ready(self) -> None:
self._ensure_engine()
def local_session(self) -> str:
engine = self._ensure_engine()
return f"{self._hostname}:{engine.get_rpc_port()}"
def register_memory(self, tensor: torch.Tensor) -> int:
return self._ensure_engine().batch_register_memory(
[tensor.data_ptr()], [tensor.nbytes]
)
def unregister_memory(self, tensor: torch.Tensor) -> bool:
engine = self._ensure_engine()
address = tensor.data_ptr()
ret = engine.unregister_memory(address)
with self._registration_lock:
if ret != 0:
logger.error(
"Mooncake EC memory unregistration failed for address %d: %d",
address,
ret,
)
self._pending_unregister[address] = tensor
return False
self._pending_unregister.pop(address, None)
return True
@staticmethod
def _source_range(tensor: torch.Tensor) -> tuple[int, int]:
# Encoder batches commonly split one storage into sibling tensor views.
# Registering each view's exact bytes avoids overlapping memory regions.
return tensor.data_ptr(), tensor.nbytes
def acquire_sources(self, tensors: list[torch.Tensor]) -> list[int]:
ranges: dict[int, tuple[int, torch.Tensor]] = {}
for tensor in tensors:
address, nbytes = self._source_range(tensor)
ranges.setdefault(address, (nbytes, tensor))
engine = self._ensure_engine()
acquired: list[int] = []
new_addresses: list[int] = []
new_lengths: list[int] = []
with self._registration_lock:
for address, (nbytes, tensor) in ranges.items():
entry = self._source_registrations.get(address)
if entry is not None:
if entry.nbytes != nbytes:
raise RuntimeError(
"Mooncake EC source storage changed size while registered"
)
entry.users += 1
acquired.append(address)
continue
new_addresses.append(address)
new_lengths.append(nbytes)
self._source_registrations[address] = _SourceRegistration(
tensor=tensor,
nbytes=nbytes,
)
acquired.append(address)
if new_addresses:
ret = engine.batch_register_memory(new_addresses, new_lengths)
if ret != 0:
for address in acquired:
entry = self._source_registrations[address]
entry.users -= 1
if entry.users == 0:
del self._source_registrations[address]
raise RuntimeError("Mooncake EC source registration failed")
return acquired
def release_sources(self, addresses: list[int]) -> bool:
if not addresses:
return True
with self._registration_lock:
unused = []
for address in addresses:
entry = self._source_registrations.get(address)
if entry is None:
continue
entry.users -= 1
if entry.users == 0:
unused.append(address)
if not unused:
return True
ret = self._ensure_engine().batch_unregister_memory(unused)
if ret != 0:
logger.warning(
"Keeping %d EC source tensors registered after Mooncake "
"unregistration failure",
len(unused),
)
return False
for address in unused:
del self._source_registrations[address]
self._pending_unregister.pop(address, None)
return True
def write(
self,
session: str,
sources: list[int],
destinations: list[int],
lengths: list[int],
) -> None:
"""Write one synchronous batch, returning only at terminal status."""
ret = self._ensure_engine().batch_transfer_sync_write(
session, sources, destinations, lengths
)
if ret != 0:
raise RuntimeError(
f"Mooncake EC push to {session} failed with status {ret}"
)
def close(self) -> None:
if self._closed:
return
self._closed = True
engine = self._engine
if engine is None:
return
with self._registration_lock:
addresses = list(self._source_registrations)
addresses.extend(self._pending_unregister)
if not addresses:
return
ret = engine.batch_unregister_memory(list(dict.fromkeys(addresses)))
if ret != 0:
logger.error("Mooncake EC batch memory unregistration failed: %d", ret)
return
self._source_registrations.clear()
self._pending_unregister.clear()
@@ -1,988 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worker-side orchestration of Mooncake control, memory, and data planes.
Consumer Workers expose rank-local reservations and publish received tensors.
Producer Workers reserve every destination shard, bind computed sources, run
batched Mooncake writes, and report asynchronous completion to the Scheduler.
"""
from __future__ import annotations
import math
import threading
import time
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from functools import partial
from typing import TYPE_CHECKING, Any, TypeVar, cast
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
_RESERVATION_TTL_SECONDS,
MooncakeECConfig,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
ConsumerControlServer,
ControlClient,
make_cancel_request,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
ConsumerMemoryPool,
ProducerMemoryPool,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeConnectorMetadata,
ECMooncakePushSpec,
ECMooncakeWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.producer import (
ProducerPushManager,
ProducerPushRecord,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.reservation import (
ConsumerReservationManager,
ConsumerReservationState,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
MooncakeTransfer,
ensure_mooncake_available,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
build_embedding_namespace,
)
from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip
logger = init_logger(__name__)
_T = TypeVar("_T")
if TYPE_CHECKING:
from vllm.config import VllmConfig
from ..mooncake_store_embedding.backend import (
MooncakeEmbeddingStoreBackend,
)
_RESERVATION_REFRESH_SECONDS = _RESERVATION_TTL_SECONDS / 2
_MAX_CANCELLED_TRANSFER_IDS = 1 << 16
_CANCEL_ATTEMPTS = 2
_TRANSFER_WORKERS = 4
_CONTROL_WORKERS = 8
class _FanoutError(RuntimeError):
"""Retain shard outcomes after every started task settles."""
def __init__(self, error: BaseException, results: list[Any | None]) -> None:
self.results = results
super().__init__(str(error))
class ECMooncakeWorker:
"""Orchestrate consumer reservations and producer push batches.
Consumers may use TP, PP, and DP. Only the first PP stage receives encoder
outputs, and each TP rank exposes a consecutive control port and receives
the same source concurrently. Producers remain unsharded and unreplicated.
With DP, the caller must route both halves of a request to the same replica
and pass that replica's control address to the producer.
"""
def __init__(self, vllm_config: VllmConfig) -> None:
ensure_mooncake_available()
config = MooncakeECConfig.from_vllm_config(vllm_config)
self._store_config = (
(
build_embedding_namespace(vllm_config),
config.store_max_pending_items,
config.store_max_pending_bytes,
config.store_read_buffer_bytes,
)
if config.cross_encoder_cache
else None
)
self._output_store: MooncakeEmbeddingStoreBackend | None = None
self.is_producer = config.is_producer
self.is_consumer = config.is_consumer
self._buffer_device = config.buffer_device
self._control_host = config.control_host
self._control_port = config.control_port
self._transfer = MooncakeTransfer(get_ip(), config.protocol)
self._consumer_memory = ConsumerMemoryPool(
config.pool_size,
self._transfer,
)
self._reservations = ConsumerReservationManager(
self._consumer_memory,
_RESERVATION_TTL_SECONDS,
_MAX_CANCELLED_TRANSFER_IDS,
)
self._consumer_rank_resolved = False
self._is_receiving_rank = True
self._tp_rank = 0
self._tp_size = 1
self._control_server: ConsumerControlServer | None = None
# Worker producer
self._producer_memory = ProducerMemoryPool(
config.pool_size,
self._transfer,
)
self._control_client = ControlClient(config.control_timeout_ms)
self._shutdown_drain_timeout_s = config.control_timeout_ms / 1000
self._io_executor = ThreadPoolExecutor(
max_workers=_TRANSFER_WORKERS,
thread_name_prefix="ec-mooncake-transfer",
)
self._control_executor = ThreadPoolExecutor(
max_workers=_CONTROL_WORKERS,
thread_name_prefix="ec-mooncake-control",
)
self._shard_pool: ThreadPoolExecutor | None = None
self._shard_pool_lock = threading.Lock()
self._push_ready = threading.Event()
self._producer_pushes = ProducerPushManager(self._push_ready.set)
self._dispatch_stop = threading.Event()
self._dispatcher: threading.Thread | None = None
self._failed_saves: set[str] = set()
self._collecting_sources = False
self._completed_loads: set[str] = set()
self._failed_loads: set[str] = set()
self._shutdown = False
if self.is_producer:
self._dispatcher = threading.Thread(
target=self._dispatch_pushes, name="ec-mooncake-ready", daemon=True
)
self._dispatcher.start()
def _dispatch_pushes(self) -> None:
pending_event = False
while not self._dispatch_stop.is_set():
self._push_ready.wait(timeout=0.001 if pending_event else None)
self._push_ready.clear()
if self._dispatch_stop.is_set():
break
if self._collecting_sources:
pending_event = False
continue
pending_event = self._producer_pushes.submit_batches(
self._io_executor, self._push_batch
)
def _resolve_consumer_rank(self) -> None:
"""Place this worker in the consumer receive topology."""
if self._consumer_rank_resolved:
return
self._consumer_rank_resolved = True
try:
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
tp_group = get_tp_group()
self._tp_rank = tp_group.rank_in_group
self._tp_size = tp_group.world_size
self._is_receiving_rank = get_pp_group().is_first_rank
except AssertionError:
# Groups are only absent outside a distributed run, where this
# worker is the whole consumer.
self._tp_rank = 0
self._tp_size = 1
self._is_receiving_rank = True
def start_services(self) -> None:
if self._store_config is not None and self._output_store is None:
from ..mooncake_store_embedding.backend import (
MooncakeEmbeddingStoreBackend,
)
from ..mooncake_store_embedding.store_client import (
create_mooncake_embedding_store_client,
)
namespace, max_items, max_bytes, read_bytes = self._store_config
self._output_store = MooncakeEmbeddingStoreBackend(
create_mooncake_embedding_store_client(read_buffer_bytes=read_bytes),
namespace,
max_pending_items=max_items,
max_pending_bytes=max_bytes,
)
if not self.is_consumer or self._control_server is not None:
return
self._resolve_consumer_rank()
if not self._is_receiving_rank:
# Later pipeline stages hold no encoder outputs, so they need
# neither a receive pool nor a control channel.
return
self._consumer_memory.prepare(torch.device(self._buffer_device))
consumer_pool = self._consumer_memory.tensor
if consumer_pool is None:
raise RuntimeError(
"Mooncake push mode requires a registered consumer buffer pool."
)
base_port = self._control_port
self._control_server = ConsumerControlServer(
# The reservation channel hands out registered-memory addresses
# and takes cancellations, so it listens where the Consumer
# advertises itself (`ec_ip`), not on every interface.
self._control_host,
base_port + self._tp_rank,
self._reserve_push_destination,
self._push_status,
self._reservations.complete,
self._reservations.cancel,
self._reservations.expire,
peer_ports=[base_port + rank for rank in range(self._tp_size)],
device=consumer_pool.device,
drain_ready=self._reservations.drain_ready,
)
try:
self._control_server.start()
except Exception:
self._control_server.close()
self._control_server = None
raise
def _reserve_push_destination(self, payload: dict[str, Any]) -> dict[str, Any]:
transfer_id = str(payload["transfer_id"])
mm_hash = str(payload["mm_hash"])
nbytes = int(payload["nbytes"])
shape = tuple(int(value) for value in payload["shape"])
dtype_name = str(payload["dtype"])
dtype = getattr(torch, dtype_name, None)
if not isinstance(dtype, torch.dtype):
raise ValueError(f"Unsupported torch dtype string: {dtype_name!r}")
expected_nbytes = math.prod(shape) * dtype.itemsize
if expected_nbytes != nbytes:
raise ValueError("shape and dtype do not match nbytes")
self._reservations.expire()
reservation, should_write = self._reservations.reserve(
transfer_id, mm_hash, nbytes, shape, dtype_name, dtype
)
if reservation is None:
raise RuntimeError("EC consumer buffer pool is full")
if reservation.state in {
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.CANCELLED,
}:
return {
"reservation_id": "",
"dst_session": "",
"dst_ptr": 0,
"nbytes": nbytes,
"write": False,
"ready": False,
"cancelled": True,
}
assert reservation.allocation is not None
return {
"reservation_id": reservation.reservation_id,
"dst_session": self._transfer.local_session(),
"dst_ptr": reservation.allocation.tensor.data_ptr(),
"nbytes": reservation.allocation.tensor.nbytes,
"write": should_write,
"ready": reservation.state is ConsumerReservationState.READY,
"cached": reservation.lease is not None,
}
def _push_status(self, transfer_id: str) -> dict[str, Any] | None:
reservation = self._reservations.status(transfer_id)
if reservation is None:
return None
assert reservation.allocation is not None
return {
"mm_hash": reservation.mm_hash,
"ready": reservation.state is ConsumerReservationState.READY,
"reservation_id": reservation.reservation_id,
"nbytes": reservation.allocation.tensor.nbytes,
"shape": list(reservation.shape),
"dtype": reservation.dtype,
}
def _shard_executor(self) -> ThreadPoolExecutor:
"""Use a separate pool so nested shard fan-out cannot deadlock."""
with self._shard_pool_lock:
if self._shard_pool is None:
self._shard_pool = ThreadPoolExecutor(
max_workers=32, thread_name_prefix="ec-mooncake-shard"
)
return self._shard_pool
def _reserve_one(self, addr: str, spec: ECMooncakePushSpec) -> dict[str, Any]:
result = self._control_client.request(
addr,
{
"op": "reserve",
"transfer_id": spec.transfer_id,
"mm_hash": spec.mm_hash,
"nbytes": spec.nbytes,
"shape": list(spec.shape),
"dtype": spec.dtype,
},
)
if not isinstance(result, dict):
raise RuntimeError("Invalid EC reservation response")
result["_received_at"] = time.monotonic()
result["addr"] = addr
return result
def _run_fanout(
self,
tasks: list[Callable[[], _T]],
on_submit: Callable[[int, Future[_T]], None] | None = None,
) -> list[_T]:
"""Run every started shard task and retain partial results on failure.
Waiting for all submitted tasks is what makes source-memory release and
partial-reservation cleanup safe after one shard fails.
"""
if not tasks:
return []
futures: list[tuple[int, Future[_T]]] = []
results: list[_T | None] = [None] * len(tasks)
error: BaseException | None = None
for index, task in enumerate(tasks[1:], 1):
try:
future = self._shard_executor().submit(task)
except Exception as exc:
error = exc
break
futures.append((index, future))
if on_submit is not None:
on_submit(index, future)
if error is None:
try:
results[0] = tasks[0]()
except Exception as exc:
error = exc
for index, future in futures:
try:
results[index] = future.result()
except Exception as exc:
if error is None:
error = exc
if error is not None:
raise _FanoutError(error, results)
return cast(list[_T], results)
def _cancel_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
*,
refresh: bool = False,
record: ProducerPushRecord | None = None,
) -> None:
reservations = [
shard for shard in reservations if not shard.get("cancelled", False)
]
if not reservations:
return
def cancel(shard: dict[str, Any]) -> dict[str, Any]:
addr = shard.get("addr")
if not isinstance(addr, str) or not addr:
raise RuntimeError(
"EC reservation is missing a confirmed shard address"
)
result = self._control_client.request(
addr,
make_cancel_request(
spec.transfer_id,
str(shard.get("reservation_id", "")),
abandon=True,
refresh=refresh,
),
)
if not isinstance(result, dict) or not result.get("cancelled"):
raise RuntimeError(
f"Could not cancel EC reservation for mm_hash={spec.mm_hash}"
)
return shard
def track(_index: int, future: Future[dict[str, Any]]) -> None:
if record is not None:
self._producer_pushes.track_shard_futures([record], [future])
self._run_fanout([partial(cancel, shard) for shard in reservations], track)
def _retry_cancel_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
*,
record: ProducerPushRecord | None = None,
) -> None:
pending = [shard for shard in reservations if not shard.get("cancelled", False)]
error: _FanoutError | None = None
for _ in range(_CANCEL_ATTEMPTS):
try:
self._cancel_reservations(spec, pending, record=record)
except _FanoutError as exc:
error = exc
pending = [
shard
for index, shard in enumerate(pending)
if exc.results[index] is None
]
continue
return
assert error is not None
raise error
def _reserve_remote(self, spec: ECMooncakePushSpec) -> list[dict[str, Any]]:
"""Reserve a destination on every shard of the consumer."""
result = self._reserve_remote_many([spec])[0]
if isinstance(result, Exception):
raise result
return result
def _refresh_remote_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
record: ProducerPushRecord | None = None,
) -> list[dict[str, Any]]:
stale = [
shard
for shard in reservations
if not shard.get("ready", False)
and not shard.get("cached", False)
and not shard.get("cancelled", False)
]
try:
self._cancel_reservations(spec, stale, refresh=True, record=record)
except _FanoutError as exc:
pending = [
shard for index, shard in enumerate(stale) if exc.results[index] is None
]
try:
self._retry_cancel_reservations(spec, pending, record=record)
except _FanoutError as cleanup_error:
raise exc from cleanup_error
raise
return self._reserve_remote(spec)
@staticmethod
def _validate_push_source(push: ProducerPushRecord) -> None:
tensor = push.source_tensor
assert tensor is not None
spec = push.spec
if tuple(tensor.shape) != tuple(spec.shape):
raise ValueError(f"EC source shape mismatch for mm_hash={spec.mm_hash}")
if str(tensor.dtype).split(".")[-1] != spec.dtype:
raise ValueError(f"EC source dtype mismatch for mm_hash={spec.mm_hash}")
if not tensor.is_contiguous():
raise ValueError(f"EC source must be contiguous for mm_hash={spec.mm_hash}")
if tensor.nbytes != spec.nbytes:
raise ValueError(f"EC source size mismatch for mm_hash={spec.mm_hash}")
def start_save_caches(
self,
metadata: ECMooncakeConnectorMetadata,
encoder_cache: dict[str, torch.Tensor] | None = None,
**kwargs: Any,
) -> None:
self._collecting_sources = True
new: dict[str, list[ProducerPushRecord]] = {}
for spec in metadata.pushes:
try:
record, created = self._producer_pushes.reserve(spec, Future)
if created:
new.setdefault(spec.consumer_zmq, []).append(record)
except ValueError:
logger.warning("Rejected conflicting EC push", exc_info=True)
self._failed_saves.add(spec.request_id)
for records in new.values():
future = self._control_executor.submit(self._reserve_batch, records)
future.add_done_callback(partial(self._reservation_batch_done, records))
if not isinstance(encoder_cache, dict):
return
for mm_hash in dict.fromkeys(spec.mm_hash for spec in metadata.pushes):
tensor = encoder_cache.get(mm_hash)
if tensor is not None:
self._bind_push_source(tensor, mm_hash)
if self._output_store is not None:
loaded = self._output_store.resolve_inputs(
metadata.store_candidates, encoder_cache, self._buffer_device
)
# Loaded entries precede the runner's new-output snapshot, so they
# need explicit P2P source binding here.
for mm_hash in loaded:
self._bind_push_source(encoder_cache[mm_hash], mm_hash)
def _reserve_batch(self, records: list[ProducerPushRecord]) -> None:
"""Resolve independent item futures with one reserve RPC per shard."""
if len(records) == 1:
record = records[0]
try:
record.reservation_future.set_result(self._reserve_remote(record.spec))
except Exception as exc:
record.reservation_future.set_exception(exc)
return
outcomes = self._reserve_remote_many([record.spec for record in records])
self._producer_pushes.finish_reservations(list(zip(records, outcomes)))
def _reserve_remote_many(
self, specs: list[ECMooncakePushSpec]
) -> list[list[dict[str, Any]] | Exception]:
shards = None
for _ in range(_CANCEL_ATTEMPTS):
shards = self._control_client.discover_shards(specs[0].consumer_zmq)
if shards is not None:
break
if shards is None:
raise RuntimeError(
f"Could not discover every EC consumer shard at {specs[0].consumer_zmq}"
)
def reserve(addr: str) -> list[dict[str, Any]]:
if len(specs) == 1:
return [{"ok": True, "result": self._reserve_one(addr, specs[0])}]
response = self._control_client.request(
addr,
{
"op": "reserve_batch",
"items": [
{
"transfer_id": spec.transfer_id,
"mm_hash": spec.mm_hash,
"nbytes": spec.nbytes,
"shape": list(spec.shape),
"dtype": spec.dtype,
}
for spec in specs
],
},
)
items = response["items"]
if len(items) != len(specs):
raise RuntimeError("Malformed EC reservation batch response")
for item in items:
if item.get("ok"):
item["result"]["_received_at"] = time.monotonic()
return items
fanout_error = None
results: list[Any]
try:
results = self._run_fanout([partial(reserve, addr) for addr in shards])
except _FanoutError as exc:
results = exc.results
fanout_error = exc
outcomes: list[list[dict[str, Any]] | Exception] = []
for index, spec in enumerate(specs):
reservations: list[dict[str, Any]] = []
error = None
for addr, items in zip(shards, results):
item = items[index] if items is not None else {}
if item.get("ok"):
reservations.append({**item["result"], "addr": addr})
else:
error = fanout_error or RuntimeError(item["error"])
reservations.append({"addr": addr, "reservation_id": ""})
if error is None:
outcomes.append(reservations)
else:
failure = _FanoutError(error, list(reservations))
try:
self._retry_cancel_reservations(spec, reservations)
except Exception as cleanup_error:
failure.__cause__ = cleanup_error
logger.exception("Failed to release partial EC reservation batch")
outcomes.append(failure)
return outcomes
@staticmethod
def _reservation_batch_done(
records: list[ProducerPushRecord], future: Future[None]
) -> None:
error = future.exception()
if error is not None:
for record in records:
if not record.reservation_future.done():
record.reservation_future.set_exception(error)
def start_load_caches(
self,
metadata: ECMooncakeConnectorMetadata,
encoder_cache: dict[str, torch.Tensor],
**kwargs: Any,
) -> None:
self._resolve_consumer_rank()
if not self._is_receiving_rank:
# Later pipeline stages never gather multimodal embeddings.
return
self._transfer.ensure_ready()
if self._buffer_device == "cuda" and not torch.accelerator.is_available():
raise RuntimeError(
"ECMooncakeConnector requires CUDA for ec_buffer_device=cuda"
)
self._reservations.retire_stale(encoder_cache, metadata.freed)
for spec in metadata.loads:
if spec.mm_hash in encoder_cache:
if not spec.local:
# The spec's id is one shard's; cancel by transfer.
self._reservations.cancel(spec.transfer_id, "")
self._completed_loads.add(spec.mm_hash)
continue
if spec.local:
tensor = self._consumer_memory.take_resident(
spec.mm_hash, tuple(spec.shape), spec.dtype
)
else:
try:
allocation = self._reservations.take(spec.transfer_id, spec.mm_hash)
tensor = allocation.tensor
except RuntimeError as e:
logger.warning("EC Mooncake pushed load failed: %s", e)
tensor = None
if tensor is None:
self._failed_loads.add(spec.mm_hash)
else:
encoder_cache[spec.mm_hash] = tensor
self._completed_loads.add(spec.mm_hash)
def _push_batch(self, pushes: list[ProducerPushRecord]) -> None:
started_at = time.monotonic()
ready: list[tuple[ProducerPushRecord, dict[str, Any]]] = []
written_pushes: dict[str, ProducerPushRecord] = {}
failure: Exception | None = None
try:
for push in pushes:
self._validate_push_source(push)
reservations = self._producer_pushes.resolve_reservations(push)
stale = [
index
for index, shard in enumerate(reservations)
if not shard.get("ready", False)
and not shard.get("cancelled", False)
and time.monotonic() - float(shard.get("_received_at", started_at))
>= _RESERVATION_REFRESH_SECONDS
]
if stale:
reservations = self._refresh_remote_reservations(
push.spec, reservations, push
)
self._producer_pushes.replace_reservations(push, reservations)
self._producer_pushes.begin_writing(push)
writable = [
shard
for shard in reservations
if not shard.get("cached", False)
and not shard.get("cancelled", False)
and shard.get("write", True)
]
source = push.source_tensor
assert source is not None
for shard in writable:
if int(shard["nbytes"]) != source.nbytes:
raise RuntimeError(
"Reserved EC size does not match tensor for "
f"mm_hash={push.spec.mm_hash}"
)
ready.append((push, shard))
written_pushes.setdefault(push.spec.transfer_id, push)
if ready:
# Stage each source once, then write it to every destination.
source_index = {
push.spec.transfer_id: index
for index, push in enumerate(written_pushes.values())
}
tensors = [
cast(torch.Tensor, push.source_tensor)
for push in written_pushes.values()
if push.source_tensor is not None
]
lengths = [tensor.nbytes for tensor in tensors]
staged = self._producer_memory.stage(tensors)
registered_sources: list[int] = []
if staged is not None:
sources = staged.tensors
else:
sources = tensors
registered_sources = self._transfer.acquire_sources(tensors)
addresses = [tensor.data_ptr() for tensor in sources]
try:
by_session: dict[str, list[tuple[int, int]]] = {}
session_records: dict[str, dict[str, ProducerPushRecord]] = {}
for push, shard in ready:
session = str(shard["dst_session"])
by_session.setdefault(session, []).append(
(source_index[push.spec.transfer_id], int(shard["dst_ptr"]))
)
session_records.setdefault(session, {})[
push.spec.transfer_id
] = push
def write(session: str, items: list[tuple[int, int]]) -> None:
self._transfer.write(
session,
[addresses[index] for index, _ in items],
[dst for _, dst in items],
[lengths[index] for index, _ in items],
)
sessions = list(by_session.items())
# Write shards concurrently to avoid serial TP latency.
def track_write(index: int, future: Future[None]) -> None:
session = sessions[index][0]
self._producer_pushes.track_shard_futures(
list(session_records[session].values()), [future]
)
writes: list[Callable[[], None]] = [
partial(write, *session) for session in sessions
]
self._run_fanout(writes, track_write)
finally:
if staged is not None:
self._producer_memory.release(staged)
self._transfer.release_sources(registered_sources)
self._producer_pushes.begin_notifying(pushes)
self._notify_completions(ready)
self._producer_pushes.complete(pushes)
except Exception as exc:
# Report asynchronously; raising here would fail EngineCore.
failure = exc
logger.exception(
"EC Mooncake push batch failed for mm_hashes=%s",
[push.spec.mm_hash for push in pushes],
)
self._producer_pushes.settle_all(pushes)
self._abandon_pushes(pushes)
finally:
if failure is not None:
self._producer_pushes.fail(pushes, failure)
def _notify_completions(
self, notifications: list[tuple[ProducerPushRecord, dict[str, Any]]]
) -> None:
"""Tell the consumer, in one message per destination, what landed."""
if not notifications:
return
by_destination: dict[str, list[tuple[ProducerPushRecord, dict[str, Any]]]] = {}
for push, reservation in notifications:
by_destination.setdefault(
str(reservation.get("addr", push.spec.consumer_zmq)), []
).append((push, reservation))
destinations = list(by_destination.items())
def notify(
consumer_zmq: str,
items: list[tuple[ProducerPushRecord, dict[str, Any]]],
) -> None:
result = self._control_client.request(
consumer_zmq,
{
"op": "complete_batch",
"items": [
{
"transfer_id": push.spec.transfer_id,
"reservation_id": reservation["reservation_id"],
}
for push, reservation in items
],
},
)
completions = result.get("items", []) if isinstance(result, dict) else []
if len(completions) != len(items):
raise RuntimeError("Malformed EC completion response")
for (push, _), completion in zip(items, completions):
if not completion.get("completed"):
raise RuntimeError(
f"Unknown EC reservation for mm_hash={push.spec.mm_hash}"
)
def track(index: int, future: Future[None]) -> None:
records = {
push.spec.transfer_id: push for push, _ in destinations[index][1]
}
self._producer_pushes.track_shard_futures(list(records.values()), [future])
self._run_fanout(
[
partial(notify, destination, items)
for destination, items in destinations
],
track,
)
@staticmethod
def _known_reservations(record: ProducerPushRecord) -> list[dict[str, Any]]:
if record.reservations:
return list(record.reservations)
try:
return list(record.reservation_future.result())
except _FanoutError as exc:
return [result for result in exc.results if isinstance(result, dict)]
except Exception:
return []
def _abandon_pushes(self, pushes: list[ProducerPushRecord]) -> None:
"""Release the consumer-side reservations of a batch that failed."""
for push in pushes:
shards = self._known_reservations(push)
if not shards:
# No confirmed topology means there is no safe address to
# cancel. The original push failure remains observable via
# ProducerPushManager.fail below.
continue
try:
self._retry_cancel_reservations(push.spec, shards, record=push)
except _FanoutError:
logger.exception(
"Failed to abandon EC reservations for transfer_id=%s",
push.spec.transfer_id,
)
def _flush_pending_pushes(self) -> None:
self._producer_pushes.submit_batches(
self._io_executor,
self._push_batch,
)
def _bind_push_source(self, tensor: torch.Tensor, mm_hash: str) -> None:
ready_event = None
if tensor.device.type == "cuda":
ready_event = torch.Event()
ready_event.record(torch.accelerator.current_stream(tensor.device))
self._producer_pushes.bind_source(mm_hash, tensor, ready_event)
def _cancel_orphaned_reservation(self, record: ProducerPushRecord) -> None:
resolution_error: Exception | None = None
try:
reservations = self._producer_pushes.resolve_reservations(record)
except Exception as exc:
resolution_error = exc
reservations = self._known_reservations(record)
reservations = [
shard for shard in reservations if not shard.get("cancelled", False)
]
if not reservations:
if resolution_error is not None:
self._producer_pushes.fail([record], resolution_error)
else:
self._producer_pushes.finish_cancel(record)
return
try:
self._retry_cancel_reservations(record.spec, reservations, record=record)
except Exception as cleanup_error:
if resolution_error is not None:
combined_error = RuntimeError(
f"EC reservation resolution failed ({resolution_error}); "
f"cleanup also failed ({cleanup_error})"
)
combined_error.__cause__ = resolution_error
cleanup_error = combined_error
self._producer_pushes.fail([record], cleanup_error)
return
if resolution_error is not None:
self._producer_pushes.fail([record], resolution_error)
else:
self._producer_pushes.finish_cancel(record)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
if not self.is_producer:
return None, None
if self._output_store is not None:
self._output_store.reap()
for record in self._producer_pushes.cancel_requests(finished_req_ids):
self._producer_pushes.submit_cancel(
record,
self._io_executor,
self._cancel_orphaned_reservation,
)
return None, None
def save_caches(
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs: Any
) -> None:
if not self.is_producer:
return
tensor = encoder_cache[mm_hash]
self._bind_push_source(tensor, mm_hash)
if self._output_store is not None:
self._output_store.save_output(mm_hash, tensor)
def build_connector_worker_meta(self) -> ECMooncakeWorkerMetadata | None:
if self.is_consumer and not self._is_receiving_rank:
# `loaded` is intersected across reporting ranks, so a stage that
# never loads must not report at all rather than report nothing.
return None
self._collecting_sources = False
self._push_ready.set()
self._flush_pending_pushes()
failures = self._producer_pushes.poll()
for mm_hash, error in failures:
logger.error(
"EC Mooncake async save failed for mm_hash=%s: %s",
mm_hash,
error,
)
reclaimed = self._consumer_memory.drain_reclaimed()
meta = ECMooncakeWorkerMetadata(
loaded=self._completed_loads,
failed_loads=self._failed_loads,
reclaimed=reclaimed,
pending_saves=self._producer_pushes.pending,
failed_saves=self._failed_saves,
)
self._failed_saves = set()
self._completed_loads = set()
self._failed_loads = set()
return meta
def close(self) -> None:
if self._shutdown:
return
self._shutdown = True
dispatcher = getattr(self, "_dispatcher", None)
if dispatcher is not None:
self._dispatch_stop.set()
self._push_ready.set()
dispatcher.join()
if self._control_server is not None:
self._reservations.begin_shutdown()
self._flush_pending_pushes()
for record in self._producer_pushes.cancel_requests(None):
self._producer_pushes.submit_cancel(
record,
self._io_executor,
self._cancel_orphaned_reservation,
)
self._control_executor.shutdown(wait=True)
self._producer_pushes.submit_batches(
self._io_executor, self._push_batch, wait=True
)
self._io_executor.shutdown(wait=True)
if self._shard_pool is not None:
self._shard_pool.shutdown(wait=True)
# Every producer-side thread that could hold a control socket is stopped.
self._control_client.close()
drained = True
if self._control_server is not None:
drained = self._reservations.wait_for_writers(
self._shutdown_drain_timeout_s
)
self._control_server.close()
if drained:
self._consumer_memory.close()
else:
logger.error(
"Timed out waiting for Mooncake EC writers; keeping the consumer "
"receive pool registered"
)
self._producer_memory.close()
self._transfer.close()
if self._output_store is not None:
self._output_store.shutdown()
self._output_store = None
@@ -1,3 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake Store support for cross-Encoder output reuse."""
@@ -1,214 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Encoder-side Mooncake Store resolver and publisher."""
from __future__ import annotations
import threading
import traceback
from collections.abc import Mapping
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
import torch
from vllm.logger import init_logger
from .data import TensorSpec, make_embedding_key
from .store_client import (
EmbeddingStoreError,
EmbeddingStoreOperationError,
MooncakeEmbeddingStoreClient,
)
logger = init_logger(__name__)
@dataclass
class _StoreSave:
tensor: torch.Tensor | None
ready_event: torch.Event | None
storage_ptr: int
def release(self) -> None:
self.tensor = None
self.ready_event = None
class MooncakeEmbeddingStoreBackend:
"""Resolve immutable encoder outputs and publish misses asynchronously."""
def __init__(
self,
store_client: MooncakeEmbeddingStoreClient,
namespace: str,
*,
max_pending_items: int,
max_pending_bytes: int,
) -> None:
self.store_client = store_client
self.namespace = namespace
self.max_pending_items = max_pending_items
self.max_pending_bytes = max_pending_bytes
self._executor = ThreadPoolExecutor(
max_workers=1,
thread_name_prefix="ec-mooncake-store",
)
self._pending: dict[str, tuple[Future[None], _StoreSave]] = {}
self._pending_storages: dict[int, tuple[torch.UntypedStorage, int]] = {}
self._pending_lock = threading.Lock()
self._pending_bytes = 0
self._step_candidates: set[str] = set()
self._closed = False
self._fatal_error: BaseException | None = None
def save_output(self, identifier: str, tensor: torch.Tensor) -> None:
if identifier in self._step_candidates:
self.reap()
self._enqueue_save(identifier, tensor)
def resolve_inputs(
self,
expected_tensors: Mapping[str, TensorSpec],
encoder_cache: dict[str, torch.Tensor],
device: torch.device | str,
) -> set[str]:
"""Load outputs before encoding; unresolved items retain normal computation."""
self.reap()
self._step_candidates = set(expected_tensors)
candidates = [key for key in expected_tensors if key not in encoder_cache]
if not candidates:
return set()
pool_keys = [
make_embedding_key(self.namespace, identifier) for identifier in candidates
]
try:
exists = self.store_client.batch_exists(pool_keys)
except (EmbeddingStoreOperationError, OSError):
logger.warning(
"Mooncake embedding Store lookup failed; falling back to encoder",
exc_info=True,
)
return set()
hits = {
pool_key: expected_tensors[identifier]
for identifier, pool_key, hit in zip(
candidates, pool_keys, exists, strict=True
)
if hit
}
try:
tensors = self.store_client.load_tensors(hits, device)
except (EmbeddingStoreOperationError, OSError):
logger.warning(
"Mooncake embedding Store GET failed; falling back to encoder",
exc_info=True,
)
return set()
loaded: set[str] = set()
for identifier, pool_key in zip(candidates, pool_keys, strict=True):
if pool_key in tensors:
encoder_cache[identifier] = tensors[pool_key]
loaded.add(identifier)
return loaded
def _enqueue_save(self, identifier: str, tensor: torch.Tensor) -> bool:
"""Admit contiguous outputs, charging each retained storage once."""
if not tensor.is_contiguous():
return False
storage = tensor.untyped_storage()
storage_ptr = storage.data_ptr()
with self._pending_lock:
if self._fatal_error is not None:
raise EmbeddingStoreError(
"Store publisher has failed"
) from self._fatal_error
if self._closed or identifier in self._pending:
return False
_, ref_count = self._pending_storages.get(storage_ptr, (storage, 0))
budget_bytes = 0 if ref_count else storage.nbytes()
if (
len(self._pending) >= self.max_pending_items
or self._pending_bytes + budget_bytes > self.max_pending_bytes
):
return False
# Only admitted work creates an event. Runtime errors propagate.
save = _StoreSave(
tensor,
_record_tensor_ready_event(tensor),
storage_ptr,
)
future = self._executor.submit(
self._save, make_embedding_key(self.namespace, identifier), save
)
self._pending[identifier] = (future, save)
# Keep the storage alive until reaping so its address cannot be reused.
self._pending_storages[storage_ptr] = (storage, ref_count + 1)
self._pending_bytes += budget_bytes
return True
def _save(self, pool_key: str, save: _StoreSave) -> None:
try:
if self.store_client.exists(pool_key):
return
tensor = save.tensor
assert tensor is not None
if save.ready_event is not None:
save.ready_event.synchronize()
self.store_client.put_tensor(pool_key, tensor)
except BaseException as error:
# Returned client frames can otherwise keep tensors via a Future's
# traceback. Uncertain I/O owners remain retained by the client.
traceback.clear_frames(error.__traceback__)
raise
finally:
tensor = None
save.release()
def reap(self) -> None:
with self._pending_lock:
for identifier, (future, save) in list(self._pending.items()):
# Observe once; a racing completion is handled on the next poll.
if not future.done():
continue
try:
future.result() # Already done: never waits on Store I/O.
except (EmbeddingStoreOperationError, OSError) as error:
logger.warning("Store PUT failed for %s: %s", identifier, error)
# This failure is consumed; do not retain publisher/caller frames.
error.__traceback__ = None
except BaseException as error:
# The client may still own unsafe native-I/O buffers. Keep
# their charge and fatal result until worker termination.
if self._fatal_error is None:
self._fatal_error = error
continue
del self._pending[identifier]
storage, ref_count = self._pending_storages[save.storage_ptr]
if ref_count == 1:
del self._pending_storages[save.storage_ptr]
self._pending_bytes -= storage.nbytes()
else:
self._pending_storages[save.storage_ptr] = (storage, ref_count - 1)
fatal = self._fatal_error
if fatal is not None:
raise fatal
def shutdown(self) -> None:
with self._pending_lock:
if self._closed:
return
self._closed = True
self._executor.shutdown(wait=True, cancel_futures=False)
self.reap()
self.store_client.close()
def _record_tensor_ready_event(tensor: torch.Tensor) -> torch.Event | None:
if tensor.device.type != "cuda":
return None
event = torch.Event()
event.record(torch.accelerator.current_stream(tensor.device))
return event
@@ -1,51 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Content identities and output contracts for the shared Encoder Store."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from vllm.config import VllmConfig
EMBEDDING_CACHE_KEY_VERSION = "v3"
def make_embedding_key(namespace: str, identifier: str) -> str:
return f"{namespace}@id:{identifier}"
@dataclass(frozen=True)
class TensorSpec:
"""Contiguous output contract used by the Encoder and Store codec."""
shape: tuple[int, ...]
dtype: str
nbytes: int
def build_embedding_namespace(vllm_config: VllmConfig) -> str:
model_config = vllm_config.model_config
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
extra_config = ec_config.ec_connector_extra_config
multimodal_config = model_config.multimodal_config
encoder_hash = (
multimodal_config.compute_hash()
if multimodal_config is not None
else "encoder:default"
)
cache_prefix = str(extra_config.get("embedding_cache_prefix", ""))
model_name = str(extra_config.get("embedding_model_identity", model_config.model))
prefix = f"{cache_prefix}@" if cache_prefix else ""
return (
f"{prefix}embedding"
f"@model:{model_name}"
f"@revision:{model_config.revision or 'default'}"
f"@encoder:{encoder_hash}"
f"@dtype:{model_config.dtype}"
f"@protocol:{EMBEDDING_CACHE_KEY_VERSION}"
)
@@ -1,463 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Thin Mooncake Store client for embedding objects.
Originally adapted from vLLM PR #47302 (0d1f71f5d36c). The v3 key namespace
uses a compact header for complete 2D embeddings, not that PR's tensor format.
"""
from __future__ import annotations
import ctypes
import struct
import threading
from collections.abc import Callable, Iterator, Mapping
from contextlib import contextmanager
from enum import IntEnum
from typing import Any
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
TensorSpec,
)
from vllm.distributed.mooncake_store import MooncakeStoreConfig, setup_mooncake_store
from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip
logger = init_logger(__name__)
# A failed transfer can still use registered memory after the binding returns.
# Poisoned clients retain their owners until process exit, including if normal
# worker teardown drops the connector. New operations are rejected.
_UNSAFE_CLIENTS: list[Any] = []
# Values from Mooncake v0.3.12 / v0.3.12.post1 types.h (ErrorCode).
# Python exposes integer returns, not this enum.
class _MooncakeErrorCode(IntEnum):
NO_AVAILABLE_HANDLE = -200
INVALID_PARAMS = -600
INVALID_REPLICA = -702
REPLICA_IS_NOT_READY = -703
OBJECT_NOT_FOUND = -704
OBJECT_ALREADY_EXISTS = -705
LEASE_EXPIRED = -707
RPC_FAIL = -900
RPC_TIMEOUT = -901
TENANT_QUOTA_EXCEEDED = -1700
# For RAM batch reads and single-replica writes, these rejections occur
# before submission or after completed I/O. Transfer timeouts can leave I/O
# in flight; all other failures retain registered buffers and poison the client.
_SAFE_IO_REJECTIONS = frozenset(
{
_MooncakeErrorCode.NO_AVAILABLE_HANDLE,
_MooncakeErrorCode.REPLICA_IS_NOT_READY,
_MooncakeErrorCode.OBJECT_NOT_FOUND,
_MooncakeErrorCode.OBJECT_ALREADY_EXISTS,
_MooncakeErrorCode.RPC_FAIL,
_MooncakeErrorCode.RPC_TIMEOUT,
}
)
# Batch GET validates capacity/replicas before I/O and checks leases afterwards.
# PUT rejects tenant quotas before allocating replicas or submitting transfers.
_SAFE_GET_REJECTIONS = _SAFE_IO_REJECTIONS | {
_MooncakeErrorCode.INVALID_PARAMS,
_MooncakeErrorCode.INVALID_REPLICA,
_MooncakeErrorCode.LEASE_EXPIRED,
}
_SAFE_PUT_REJECTIONS = _SAFE_IO_REJECTIONS | {_MooncakeErrorCode.TENANT_QUOTA_EXCEEDED}
_MOONCAKE_TENSOR_OBJECT_MAGIC = 0x4D4F4F4E
# Magic, dtype, token count, hidden dimension. Format versioning is in the key.
_MOONCAKE_TENSOR_HEADER = struct.Struct("<IIQQ")
_MOONCAKE_DTYPE_TO_TORCH_DTYPE = {
0: "torch.float32",
11: "torch.float16",
12: "torch.bfloat16",
}
_TORCH_DTYPE_TO_MOONCAKE_DTYPE = {
value: key for key, value in _MOONCAKE_DTYPE_TO_TORCH_DTYPE.items()
}
def create_mooncake_embedding_store_client(
read_buffer_bytes: int = 128 * 1024**2,
) -> MooncakeEmbeddingStoreClient:
try:
from mooncake.store import ( # type: ignore
MooncakeDistributedStore,
ObjectDataType,
ReplicateConfig,
)
except ImportError as e:
raise ImportError(
"Install mooncake with Store support to enable cross_encoder_cache."
) from e
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
config = MooncakeStoreConfig.load_from_config()
if config.enable_offload:
raise ValueError(
"cross_encoder_cache supports a RAM Store; disable enable_offload"
)
store = MooncakeDistributedStore()
local_ip = get_ip()
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
setup_mooncake_store(store, config, local_hostname)
logger.info(
"Initialized embedding Mooncake store mode=%s global_segment_size=%d "
"local_buffer_size=%d",
config.mode,
config.global_segment_size,
config.local_buffer_size,
)
replicate_config = ReplicateConfig()
replicate_config.data_type = ObjectDataType.TENSOR
return MooncakeEmbeddingStoreClient(
store, replicate_config=replicate_config, read_buffer_bytes=read_buffer_bytes
)
class EmbeddingStoreError(RuntimeError):
"""A Store failure that must propagate to the worker."""
class EmbeddingStoreOperationError(EmbeddingStoreError):
"""A rejected or completed operation that permits normal Encoder fallback."""
def _batch_get_completed_safely(results: Any, capacities: list[int]) -> bool:
return (
isinstance(results, list)
and len(results) == len(capacities)
and all(
type(result) is int
and (0 <= result <= capacity or result in _SAFE_GET_REJECTIONS)
for result, capacity in zip(results, capacities, strict=True)
)
)
class MooncakeEmbeddingStoreClient:
"""Wraps Mooncake object and buffer APIs used by embedding transfer."""
def __init__(
self,
store: Any,
replicate_config: Any | None = None,
*,
read_buffer_bytes: int = 128 * 1024**2,
):
self._lifetime_lock = threading.Lock()
self._unsafe_owners: list[Any] = []
self._poisoned = False
self.store = store
self.replicate_config = replicate_config
self._read_buffer_bytes = read_buffer_bytes
self._read_buffer: torch.Tensor | None = None
self._put_header: ctypes.Array[ctypes.c_char] | None = None
def _check_healthy(self) -> None:
if self._poisoned:
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; worker must exit"
)
def _poison(self, owners: list[Any]) -> None:
with self._lifetime_lock:
self._unsafe_owners.extend(owners)
if not self._poisoned:
self._poisoned = True
_UNSAFE_CLIENTS.append(self)
@contextmanager
def _registered_io(
self,
buffers: list[tuple[Any, int, int]],
) -> Iterator[Callable[[int], None]]:
self._check_healthy()
registered = []
submitted = False
confirmed = False
def confirm(result: int) -> None:
nonlocal confirmed
if type(result) is not int or (
result < 0 and result not in _SAFE_PUT_REJECTIONS
):
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; retaining buffers"
)
confirmed = True
try:
for owner, addr, size in buffers:
ret = self.store.register_buffer(addr, size)
if ret != 0:
raise EmbeddingStoreOperationError(
f"Failed to register embedding buffer: {ret}"
)
registered.append(addr)
submitted = True
yield confirm
except BaseException as error:
if submitted and not confirmed:
self._poison([owner for owner, _, _ in buffers])
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; retaining buffers"
) from error
raise
finally:
if not submitted or confirmed:
try:
for addr in reversed(registered):
self._unregister_buffer(addr)
except BaseException:
self._poison([owner for owner, _, _ in buffers])
raise
def close(self) -> None:
"""Close the supported Mooncake binding after all I/O is drained."""
self._check_healthy()
if self._read_buffer is not None:
try:
self._unregister_buffer(self._read_buffer.data_ptr())
except BaseException:
self._poison([self._read_buffer])
raise
self._read_buffer = None
if self._put_header is not None:
try:
self._unregister_buffer(ctypes.addressof(self._put_header))
except BaseException:
self._poison([self._put_header])
raise
self._put_header = None
ret = self.store.close()
if ret != 0:
raise EmbeddingStoreError(
f"failed to close embedding Mooncake Store: {ret}"
)
def exists(self, pool_key: str) -> bool:
return self.batch_exists([pool_key])[0]
def batch_exists(self, keys: list[str]) -> list[bool]:
self._check_healthy()
if not keys:
return []
states = self.store.batch_is_exist(keys)
if len(states) != len(keys):
raise RuntimeError(
"Mooncake Store returned an unexpected number of lookup results"
)
return [state == 1 for state in states]
def load_tensors(
self, expected: Mapping[str, TensorSpec], device: torch.device | str
) -> dict[str, torch.Tensor]:
"""Synchronously load runner inputs into independently owned tensors."""
self._check_healthy()
device = torch.device(device)
loaded: dict[str, torch.Tensor] = {}
chunk: dict[str, TensorSpec] = {}
chunk_bytes = 0
for key, spec in expected.items():
size = _MOONCAKE_TENSOR_HEADER.size + spec.nbytes
if size > self._read_buffer_bytes:
logger.warning(
"Skipping Store GET for %s: object bytes=%d exceed read buffer=%d",
key,
size,
self._read_buffer_bytes,
)
continue
if chunk_bytes + size > self._read_buffer_bytes:
loaded.update(self._load_chunk(chunk, device))
chunk = {}
chunk_bytes = 0
chunk[key] = spec
chunk_bytes += size
if chunk:
loaded.update(self._load_chunk(chunk, device))
return loaded
def _load_chunk(
self, expected: dict[str, TensorSpec], device: torch.device
) -> dict[str, torch.Tensor]:
self._check_healthy()
if self._read_buffer is None:
buffer = torch.empty(
self._read_buffer_bytes,
dtype=torch.uint8,
device="cpu",
pin_memory=device.type == "cuda",
)
ret = self.store.register_buffer(buffer.data_ptr(), buffer.nbytes)
if ret != 0:
raise EmbeddingStoreOperationError(
f"Failed to register embedding read buffer: {ret}"
)
self._read_buffer = buffer
buffer = self._read_buffer
base = buffer.data_ptr()
sizes = [
_MOONCAKE_TENSOR_HEADER.size + spec.nbytes for spec in expected.values()
]
offsets: list[int] = []
end = 0
for size in sizes:
offsets.append(end)
end += size
try:
results = self.store.batch_get_into(
list(expected), [base + offset for offset in offsets], sizes
)
if not _batch_get_completed_safely(results, sizes):
raise EmbeddingStoreError("Unsafe batch GET results")
except BaseException as error:
self._poison([buffer])
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; retaining read buffer"
) from error
valid: list[tuple[str, TensorSpec, int]] = []
for (key, spec), offset, size, result in zip(
expected.items(), offsets, sizes, results, strict=True
):
if result != size:
logger.warning(
"Store GET skipped for %s: result=%d, expected bytes=%d",
key,
result,
size,
)
continue
try:
_validate_mooncake_tensor_metadata(
key,
ctypes.string_at(base + offset, _MOONCAKE_TENSOR_HEADER.size),
spec,
)
except EmbeddingStoreOperationError as error:
logger.warning("Store GET skipped for %s: %s", key, error)
continue
valid.append((key, spec, offset + _MOONCAKE_TENSOR_HEADER.size))
if not valid:
return {}
# Allocate every destination before submitting any asynchronous copy.
targets = {
key: torch.empty(
spec.shape,
dtype=getattr(torch, spec.dtype.removeprefix("torch.")),
device=device,
)
for key, spec, _ in valid
}
ready = torch.Event() if device.type == "cuda" else None
try:
for key, spec, offset in valid:
targets[key].view(torch.uint8).view(-1).copy_(
buffer[offset : offset + spec.nbytes],
non_blocking=ready is not None,
)
if ready is not None:
ready.record(torch.accelerator.current_stream(device))
ready.synchronize()
except BaseException as error:
if ready is not None:
self._poison([buffer, *targets.values(), ready])
raise EmbeddingStoreError(
"Embedding copy completion is unconfirmed; retaining buffers"
) from error
raise
return targets
def put_tensor(self, pool_key: str, tensor: torch.Tensor) -> None:
self._check_healthy()
if not tensor.is_contiguous():
raise EmbeddingStoreOperationError("embedding tensor must be contiguous")
metadata = _encode_mooncake_tensor_metadata(tensor)
# The single publisher reuses this header only after PUT completion.
if self._put_header is None:
header = ctypes.create_string_buffer(len(metadata))
ret = self.store.register_buffer(ctypes.addressof(header), len(metadata))
if ret != 0:
raise EmbeddingStoreOperationError(
f"Failed to register embedding header: {ret}"
)
self._put_header = header
header_ptr = ctypes.addressof(self._put_header)
ctypes.memmove(header_ptr, metadata, len(metadata))
data_size = tensor.numel() * tensor.element_size()
buffers = [
(tensor, tensor.data_ptr(), data_size),
]
with self._registered_io(buffers) as confirm:
[result] = self.store.batch_put_from_multi_buffers(
[pool_key],
[[header_ptr, tensor.data_ptr()]],
[[len(metadata), data_size]],
self.replicate_config,
)
confirm(result)
if result < 0:
raise EmbeddingStoreOperationError(
f"failed to put embedding tensor for {pool_key}"
)
def _unregister_buffer(self, addr: int) -> None:
try:
ret = self.store.unregister_buffer(addr)
except Exception as error:
raise EmbeddingStoreError(
f"could not confirm buffer unregistration addr={addr:#x}"
) from error
if ret != 0:
raise EmbeddingStoreError(
f"failed to unregister embedding buffer addr={addr:#x}: {ret}"
)
def _validate_mooncake_tensor_metadata(
pool_key: str,
metadata: bytes,
expected: TensorSpec,
) -> None:
magic, dtype, num_tokens, hidden_dim = _MOONCAKE_TENSOR_HEADER.unpack(metadata)
if magic != _MOONCAKE_TENSOR_OBJECT_MAGIC:
raise EmbeddingStoreOperationError(
f"invalid Mooncake tensor metadata header for {pool_key}"
)
if _MOONCAKE_DTYPE_TO_TORCH_DTYPE.get(dtype) != expected.dtype:
raise EmbeddingStoreOperationError(
f"unexpected Mooncake tensor dtype for {pool_key}: {dtype}"
)
if (num_tokens, hidden_dim) != expected.shape:
raise EmbeddingStoreOperationError(
f"embedding tensor shape mismatch for {pool_key}: expected={expected.shape}"
)
def _encode_mooncake_tensor_metadata(tensor: torch.Tensor) -> bytes:
dtype = str(tensor.dtype)
if dtype not in _TORCH_DTYPE_TO_MOONCAKE_DTYPE:
raise EmbeddingStoreOperationError(
f"unsupported embedding tensor dtype: {dtype}"
)
if tensor.ndim != 2:
raise EmbeddingStoreOperationError(
f"embedding tensor must be 2D: shape={tuple(tensor.shape)}"
)
return _MOONCAKE_TENSOR_HEADER.pack(
_MOONCAKE_TENSOR_OBJECT_MAGIC,
_TORCH_DTYPE_TO_MOONCAKE_DTYPE[dtype],
*tensor.shape,
)
@@ -150,30 +150,12 @@ class KVConnectorFactory:
# The registration should not be done in each individual file, as we want to
# only load the files corresponding to the current connector.
KVConnectorFactory.register_connector(
"ExampleConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.example_connector",
"ExampleConnector",
)
KVConnectorFactory.register_connector(
"ExampleHiddenStatesConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector",
"ExampleHiddenStatesConnector",
)
KVConnectorFactory.register_connector(
"LMCacheConnectorV1",
"vllm.distributed.kv_transfer.kv_connector.v1.lmcache_connector",
"LMCacheConnectorV1",
)
KVConnectorFactory.register_connector(
"LMCacheMPConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.lmcache_mp_connector",
"LMCacheMPConnector",
)
KVConnectorFactory.register_connector(
"NixlConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
@@ -222,21 +204,6 @@ KVConnectorFactory.register_connector(
"DecodeBenchConnector",
)
KVConnectorFactory.register_connector(
"MooncakeConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector",
"MooncakeConnector",
)
KVConnectorFactory.register_connector(
"MooncakeStoreConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.connector",
"MooncakeStoreConnector",
)
KVConnectorFactory.register_connector(
"FlexKVConnectorV1",
"vllm.distributed.kv_transfer.kv_connector.v1.flexkv_connector",
"FlexKVConnectorV1",
)
KVConnectorFactory.register_connector(
"SimpleCPUOffloadConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector",
@@ -1,270 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.distributed.kv_events import KVCacheEvent
from vllm.forward_context import ForwardContext
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
# FlexKV is a distributed KV Store and multi-level cache management system for
# ultra-large-scale LLM inference.
# GitHub: https://github.com/taco-project/FlexKV
# Install: git clone git@github.com:taco-project/FlexKV.git \
# && cd FlexKV && bash build.sh
class FlexKVConnectorV1(KVConnectorBase_V1):
"""KV Connector that offloads KV cache to FlexKV.
FlexKV is a distributed KV Store and multi-level cache management system
designed for ultra-large-scale LLM inference. It supports offloading KV
cache to CPU memory, SSD, and remote storage.
Installation:
See https://github.com/taco-project/FlexKV for installation instructions.
Quick start::
git clone git@github.com:taco-project/FlexKV.git
cd FlexKV && bash build.sh
Configuration:
Pass ``kv_connector="FlexKVConnectorV1"`` via ``--kv-transfer-config``::
--kv-transfer-config \
'{"kv_connector":"FlexKVConnectorV1","kv_role":"kv_both"}'
"""
def __init__(
self,
vllm_config: "VllmConfig",
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(
vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config
)
try:
from flexkv.integration.vllm.vllm_v1_adapter import FlexKVConnectorV1Impl
except ImportError as e:
raise ImportError(
"FlexKV is not installed. Please install it to use "
"FlexKVConnectorV1. See https://github.com/taco-project/FlexKV "
"for installation instructions."
) from e
self._flexkv_connector = FlexKVConnectorV1Impl(vllm_config, role)
def shutdown(self):
self._flexkv_connector.shutdown()
# ==============================
# Worker-side methods
# ==============================
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
"""No-op for FlexKV (currently).
FlexKV manages all KV transfers on the **scheduler side** via
``build_connector_meta`` (which calls ``launch_tasks``) and
``update_connector_output`` (which polls ``query_finished_task``).
KV blocks are transferred directly between the FlexKV server and
vLLM's GPU memory without worker-side intervention during the
forward pass — similar to how NIXL operates.
These worker-side hooks are kept (rather than omitted) to satisfy
the ``KVConnectorBase_V1`` interface contract and to serve as
extension points for a future worker-side layer-pipelining path.
Args:
forward_context (ForwardContext): the forward context.
**kwargs (Any): additional arguments (unused).
"""
self._flexkv_connector.start_load_kv(forward_context, **kwargs)
def wait_for_layer_load(self, layer_name: str) -> None:
"""No-op for FlexKV (currently).
FlexKV manages all KV transfers on the scheduler side.
This hook is retained for ``KVConnectorBase_V1`` API compatibility.
Args:
layer_name: the name of the layer (unused).
"""
self._flexkv_connector.wait_for_layer_load(layer_name)
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: "AttentionMetadata",
**kwargs,
) -> None:
"""No-op for FlexKV (currently).
FlexKV offloads KV cache asynchronously from the scheduler side
after a request finishes (see ``request_finished``). It does not
intercept individual layer tensors during the forward pass.
This hook is retained to satisfy ``KVConnectorBase_V1`` and as an
extension point for future per-layer async offload support.
Args:
layer_name (str): the name of the layer (unused).
kv_layer (torch.Tensor): the paged KV buffer (unused).
attn_metadata (AttentionMetadata): the attention metadata (unused).
**kwargs (Any): additional arguments (unused).
"""
self._flexkv_connector.save_kv_layer(
layer_name, kv_layer, attn_metadata, **kwargs
)
def wait_for_save(self):
"""No-op for FlexKV (currently).
KV offload tasks are tracked asynchronously by the scheduler
connector via ``request_finished`` / ``query_finished_task``.
There is no pending worker-side save to wait for at
forward-context exit.
Retained to satisfy ``KVConnectorBase_V1`` and as an extension
point for future worker-side save-completion signalling.
"""
self._flexkv_connector.wait_for_save()
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Notify worker-side connector of requests that have finished
generating tokens.
Returns:
Tuple of (sending/saving ids, recving/loading ids) for requests
that have finished asynchronous transfer. The finished saves/sends
req ids must belong to a set provided in a call to this method
(this call or a prior one).
"""
return self._flexkv_connector.get_finished(finished_req_ids)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Initialize with the KV caches. Useful for pre-registering the
KV caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names to kv cache tensors.
"""
self._flexkv_connector.register_kv_caches(kv_caches)
# ==============================
# Scheduler-side methods
# ==============================
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int, bool]:
"""Get the number of new tokens that can be loaded from the
external KV cache beyond ``num_computed_tokens``.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally computed
tokens for this request.
Returns:
Tuple of (num_external_tokens, is_ready) where
num_external_tokens is the number of additional tokens that
can be loaded from the external KV cache.
"""
return self._flexkv_connector.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""Update KVConnector state after block allocation."""
self._flexkv_connector.update_state_after_alloc(
request, blocks, num_external_tokens
)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
return self._flexkv_connector.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput):
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
self._flexkv_connector.update_connector_output(connector_output)
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""Called when a request has finished, before its blocks are freed.
Returns:
Tuple of (async_save, kv_transfer_params) where async_save is
True if the request is being saved/sent asynchronously and blocks
should not be freed until the request_id is returned from
:meth:`get_finished`. kv_transfer_params is an optional dict of
KVTransferParams to be included in the request outputs.
"""
return self._flexkv_connector.request_finished(request, block_ids)
def take_events(self) -> Iterable["KVCacheEvent"]:
"""Collect buffered KV cache events.
Returns:
New KV cache events since the last call.
"""
return self._flexkv_connector.take_events()
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""Get the KV connector stats collected during the last interval."""
return self._flexkv_connector.get_kv_connector_stats()
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get the block ids that have failed to load."""
return self._flexkv_connector.get_block_ids_with_load_errors()
@@ -1,358 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_events import (
BlockStored,
KVCacheEvent,
KVConnectorKVEvents,
KVEventAggregator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.forward_context import ForwardContext
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
class LMCacheKVEvents(KVConnectorKVEvents):
"""Concrete implementation of KVConnectorKVEvents using KVEventAggregator."""
def __init__(self, num_workers: int) -> None:
self._aggregator = KVEventAggregator(num_workers)
def add_events(self, events: list[KVCacheEvent]) -> None:
self._aggregator.add_events(events)
def aggregate(self) -> "LMCacheKVEvents":
"""Aggregate KV events and retain only common events."""
common_events = self._aggregator.get_common_events()
self._aggregator.clear_events()
self._aggregator.add_events(common_events)
self._aggregator.reset_workers()
return self
def increment_workers(self, count: int = 1) -> None:
self._aggregator.increment_workers(count)
def get_all_events(self) -> list[KVCacheEvent]:
return self._aggregator.get_all_events()
def get_number_of_workers(self) -> int:
return self._aggregator.get_number_of_workers()
def clear_events(self) -> None:
self._aggregator.clear_events()
self._aggregator.reset_workers()
def __repr__(self) -> str:
return f"<LMCacheKVEvents events={self.get_all_events()}>"
class LMCacheConnectorV1(KVConnectorBase_V1):
@classmethod
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
"""LMCache requires PIECEWISE CUDA graph mode when layerwise
operations are enabled. The wait_for_layer_load and save_kv_layer
methods perform actual async synchronization that cannot be
captured in CUDA graphs.
"""
return extra_config.get("use_layerwise", False)
def __init__(
self,
vllm_config: "VllmConfig",
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(
vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config
)
assert vllm_config.kv_transfer_config is not None
use_native = vllm_config.kv_transfer_config.get_from_extra_config(
"use_native", False
)
if use_native:
logger.info("Initializing native LMCache connector")
# lazy import
from vllm.distributed.kv_transfer.kv_connector.v1 import lmcache_integration
_adapter = lmcache_integration.vllm_v1_adapter
cls = _adapter.LMCacheConnectorV1Impl
else:
logger.info("Initializing latest dev LMCache connector")
# lazy import
from lmcache.integration.vllm.vllm_v1_adapter import (
LMCacheConnectorV1Impl as LMCacheConnectorLatestImpl,
)
cls = LMCacheConnectorLatestImpl
self._lmcache_engine = cls(vllm_config, role, self)
self._kv_cache_events: LMCacheKVEvents | None = None
# ==============================
# Worker-side methods
# ==============================
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Initialize with the KV caches. Useful for pre-registering the
KV Caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names, kv cache
"""
if hasattr(self._lmcache_engine, "register_kv_caches"):
self._lmcache_engine.register_kv_caches(kv_caches)
else:
logger.warning(
"LMCache engine does not support register_kv_caches, "
"please check and use the latest version"
)
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
super().bind_connector_metadata(connector_metadata)
engine = self._lmcache_engine
if hasattr(engine, "bind_connector_metadata"):
engine.bind_connector_metadata(connector_metadata)
else:
# Compat for engines predating the bind hook: start_load_kv may
# run after the forward (SchedulerOutput.has_sync_kv_loads), so
# the per-step layerwise state it used to reset must be reset
# here. Remove once LMCache ships bind_connector_metadata.
if hasattr(engine, "current_layer"):
engine.current_layer = 0
if hasattr(engine, "layerwise_retrievers"):
engine.layerwise_retrievers = []
engine.layerwise_storers = []
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""Start loading the KV cache from the connector to vLLM's paged
KV buffer. This is called from the forward context before the
forward pass to enable async loading during model execution.
Args:
forward_context (ForwardContext): the forward context.
**kwargs: additional arguments for the load operation
Note:
The number of elements in kv_caches and layer_names should be
the same.
"""
self._lmcache_engine.start_load_kv(forward_context, **kwargs)
def wait_for_layer_load(self, layer_name: str) -> None:
"""Block until the KV for a specific layer is loaded into vLLM's
paged buffer. This is called from within attention layer to ensure
async copying from start_load_kv is complete.
This interface will be useful for layer-by-layer pipelining.
Args:
layer_name: the name of that layer
"""
self._lmcache_engine.wait_for_layer_load(layer_name)
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
"""Start saving the a layer of KV cache from vLLM's paged buffer
to the connector. This is called from within attention layer to
enable async copying during execution.
Args:
layer_name (str): the name of the layer.
kv_layer (torch.Tensor): the paged KV buffer of the current
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
self._lmcache_engine.save_kv_layer(
layer_name, kv_layer, attn_metadata, **kwargs
)
def wait_for_save(self):
"""Block until all the save operations is done. This is called
as the forward context exits to ensure that the async saving
from save_kv_layer is complete before finishing the forward.
This prevents overwrites of paged KV buffer before saving done.
"""
self._lmcache_engine.wait_for_save()
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Notifies worker-side connector ids of requests that have
finished generating tokens.
Returns:
ids of requests that have finished asynchronous transfer
(requests that previously returned True from request_finished()),
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
return self._lmcache_engine.get_finished(finished_req_ids)
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get the set of block IDs that failed to load.
Returns:
Set of block IDs that encountered load errors.
Empty set if no load errors occurred.
"""
method = getattr(self._lmcache_engine, "get_block_ids_with_load_errors", None)
if callable(method):
return method()
# Fallback for older versions that don't support this method
return set()
def get_kv_connector_kv_cache_events(self) -> LMCacheKVEvents | None:
"""Get the KV connector kv cache events collected during the last interval."""
events = self._lmcache_engine.get_kv_events() # type: ignore [attr-defined]
if not events:
return None
blocks: list[BlockStored] = [
BlockStored(
block_hashes=e.block_hashes,
parent_block_hash=e.parent_block_hash,
token_ids=e.token_ids,
lora_id=e.lora_id,
block_size=e.block_size,
medium=e.medium,
lora_name=getattr(e, "lora_name", None),
)
for e in events
]
lmcache_kv_events = LMCacheKVEvents(num_workers=1)
lmcache_kv_events.add_events(blocks)
return lmcache_kv_events
# ==============================
# Scheduler-side methods
# ==============================
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally
computed tokens for this request
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
return self._lmcache_engine.get_num_new_matched_tokens(
request, num_computed_tokens
), False
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""Update KVConnector state after block allocation."""
self._lmcache_engine.update_state_after_alloc(request, num_external_tokens)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
return self._lmcache_engine.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput):
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
# Get the KV events
kv_cache_events = connector_output.kv_cache_events
if not kv_cache_events or not isinstance(kv_cache_events, LMCacheKVEvents):
return
if self._kv_cache_events is None:
self._kv_cache_events = kv_cache_events
else:
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
self._kv_cache_events.increment_workers(
kv_cache_events.get_number_of_workers()
)
return
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""Called when a request has finished, before its blocks are freed.
Returns:
True if the request is being saved/sent asynchronously and blocks
should not be freed until the request_id is returned from
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
return self._lmcache_engine.request_finished(request, block_ids)
def take_events(self) -> Iterable["KVCacheEvent"]:
"""Take the KV cache events from the connector.
Yields:
New KV cache events since the last call.
"""
if self._kv_cache_events is not None:
self._kv_cache_events.aggregate()
kv_cache_events = self._kv_cache_events.get_all_events()
yield from kv_cache_events
self._kv_cache_events.clear_events()
self._kv_cache_events = None
@@ -1,20 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from . import multi_process_adapter, vllm_v1_adapter
from .multi_process_adapter import (
LMCacheMPSchedulerAdapter,
LMCacheMPWorkerAdapter,
LoadStoreOp,
ParallelStrategy,
)
__all__ = [
"vllm_v1_adapter",
"multi_process_adapter",
"LMCacheMPSchedulerAdapter",
"LMCacheMPWorkerAdapter",
"LoadStoreOp",
"ParallelStrategy",
]
@@ -1,735 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from collections.abc import Iterable
from dataclasses import dataclass
from itertools import islice
from typing import Any
import torch
import zmq
from lmcache.utils import _lmcache_nvtx_annotate, init_logger
from lmcache.v1.multiprocess.custom_types import (
CudaIPCWrapper,
IPCCacheEngineKey,
KVCache,
)
from lmcache.v1.multiprocess.mq import MessageQueueClient, MessagingFuture
from lmcache.v1.multiprocess.protocol import RequestType, get_response_class
logger = init_logger(__name__)
def wrap_kv_caches(kv_caches: dict[str, torch.Tensor]) -> KVCache:
logger.info("KV caches keys are %s", list(kv_caches.keys()))
return [CudaIPCWrapper(tensor) for tensor in kv_caches.values()]
def striding_block_hashes(
block_hashes: list[bytes], blocks_in_chunk: int
) -> Iterable[bytes]:
"""Extract chunk-level hashes from block hashes by striding.
In hash-based vLLM, each vLLM block has its own hash. LMCache chunks
span ``blocks_in_chunk`` consecutive blocks. The representative hash
for a chunk is the hash of the **last** block in that chunk (because
each block hash already encodes its prefix). So we start at index
``blocks_in_chunk - 1`` and stride by ``blocks_in_chunk``.
"""
return islice(block_hashes, blocks_in_chunk - 1, None, blocks_in_chunk)
def send_lmcache_request(
mq_client: MessageQueueClient,
request_type: RequestType,
payloads: list[Any],
) -> MessagingFuture[Any]:
"""Helper function to send the request to the LMCache multiprocess server.
Args:
mq_client: The LMCache multiprocess mode message queue client
request_type: The request type
payloads: The request payloads
Returns:
A messaging future for the request
"""
future = mq_client.submit_request(
request_type, payloads, get_response_class(request_type)
)
return future
def get_lmcache_chunk_size(
mq_client: MessageQueueClient,
) -> int:
"""Helper function to get the LMCache chunk size from the server.
Args:
mq_client: The LMCache multiprocess mode message queue client
Returns:
An integer representing the LMCache chunk size
"""
future = send_lmcache_request(mq_client, RequestType.GET_CHUNK_SIZE, [])
chunk_size = future.result()
return chunk_size
@dataclass
class ParallelStrategy:
use_mla: bool
"""Whether to use the MLA."""
kv_world_size: int
"""
The kv world size, kv_world_size may not be equal to the actual_world_size,
in the case of mla, it will 'exclude' the effect of TP, the value is
calculated by `extract_world_size_and_kv_rank` in `lmcache_mp_connector.py`.
"""
kv_worker_id: int
"""
The kv worker id of the sub-process, kv_worker_id may not be equal to the
actual_worker_id, in the case of mla, it will 'exclude' the effect of TP,
the value is calculated by `extract_world_size_and_kv_rank` in
`lmcache_mp_connector.py`.
"""
actual_world_size: int
"""The actual world size."""
actual_worker_id: int
"""The actual worker id of the sub-process."""
tp_size: int
"""The tensor parallel size."""
pp_size: int
"""The pipeline parallel size."""
@dataclass
class LoadStoreOp:
block_ids: list[int]
"""Block ids for the load/store operation"""
token_ids: list[int] | None = None
"""Token IDs for the load/store operation (token mode)"""
block_hashes: list[bytes] | None = None
"""Block hashes for the load/store operation (hash mode)"""
start: int = 0
"""Start token index (token mode only)"""
end: int = 0
"""End token index (token mode only)"""
def __len__(self) -> int:
return len(self.block_ids)
StoreResult = bool
RetrieveResult = list[bool]
LookupResult = int
class LMCacheMPSchedulerAdapter:
def __init__(
self,
server_url: str,
context: zmq.Context,
model_name: str,
vllm_block_size: int,
parallel_strategy: ParallelStrategy,
):
"""Args:
server_url: The server URL for the LMCache message queue
context: The ZMQ context
model_name: The model name used for LMCache keys
vllm_block_size: The block size used in vLLM
parallel_strategy:
The parallel strategy, which includes `use_mla`,
`world_size`, `worker_id` and so on
"""
self.mq_client = MessageQueueClient(server_url, context)
# Request futures
self.lookup_futures: dict[str, MessagingFuture[LookupResult]] = {}
self.model_name = model_name
self.parallel_strategy = parallel_strategy
# Read chunk size from lmcache
self.chunk_size = get_lmcache_chunk_size(self.mq_client)
assert self.chunk_size % vllm_block_size == 0, (
"LMCache chunk size should be a multiple of vLLM block size"
)
self.blocks_in_chunk = self.chunk_size // vllm_block_size
@property
def world_size(self) -> int:
"""The world size."""
return self.parallel_strategy.kv_world_size
@property
def worker_id(self) -> int:
"""The worker id."""
return self.parallel_strategy.kv_worker_id
@property
def tp_size(self) -> int:
"""The tensor parallel size."""
return self.parallel_strategy.tp_size
@_lmcache_nvtx_annotate
def maybe_submit_lookup_request(
self,
request_id: str,
block_hashes: list[bytes] | None = None,
token_ids: list[int] | None = None,
) -> None:
"""Submit a new lookup request to LMCache if there is no ongoing request.
Supports both token-based and hash-based vLLM:
- token_ids: token IDs (token-based vLLM) -> single token-mode key
- block_hashes: block hashes (hash-based vLLM) -> strided hash-mode keys
Exactly one of block_hashes or token_ids must be provided.
Args:
request_id: The ID of the lookup request. The same ID indicates it's
from the same request
block_hashes: Block hashes to lookup from LMCache (hash mode)
token_ids: Token IDs to lookup from LMCache (token mode)
Returns:
None
Notes:
This function will have a side-effect: submitting a look up request to
LMCache, which will essentially 'lock' the KV cache chunks in the LMCache
for later retrieve operations.
In the meantime, this function will record the lookup request, and the
status of the look up request can be checked by `check_lookup_result`.
"""
if request_id in self.lookup_futures:
# Skip if there is already a lookup request
return
assert (block_hashes is None) != (token_ids is None), (
"Exactly one of block_hashes or token_ids must be provided"
)
if block_hashes is not None:
# Hash mode: stride block hashes -> N hash-mode keys
chunk_hashes = list(
striding_block_hashes(block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
]
else:
# Token mode: truncate to chunk-aligned length
assert token_ids is not None
aligned_end = (len(token_ids) // self.chunk_size) * self.chunk_size
if aligned_end == 0:
return
keys = [
self._create_key(
token_ids,
start=0,
end=aligned_end,
request_id=request_id,
).no_worker_id_version()
]
future = send_lmcache_request(
self.mq_client,
RequestType.LOOKUP,
[keys],
)
self.lookup_futures[request_id] = future
@_lmcache_nvtx_annotate
def check_lookup_result(self, request_id: str) -> int | None:
"""Check the result of a previously submitted lookup request.
Args:
request_id: The ID of the lookup request submitted in
`maybe_submit_lookup_request`
Returns:
An integer representing the total number of tokens matched
in LMCache (prefix matching), or
None if the lookup request is not finished yet.
"""
assert request_id in self.lookup_futures, (
f"Lookup request for request_id={request_id} has not been submitted"
)
future = self.lookup_futures[request_id]
if not future.query():
return None
result = future.result()
num_chunks = result
return num_chunks * self.chunk_size
def num_blocks_per_chunk(self) -> int:
"""Returns:
The number of vllm blocks in a LMCache data chunk
"""
return self.blocks_in_chunk
def cleanup_lookup_result(self, request_id: str) -> None:
"""Clean up lookup future for a finished request to prevent memory leak.
Args:
request_id: The ID of the finished request.
"""
self.lookup_futures.pop(request_id, None)
def end_session(self, request_id: str) -> None:
"""Notify LMCache server to remove the session for a finished request.
Args:
request_id: The ID of the finished request.
"""
send_lmcache_request(
self.mq_client,
RequestType.END_SESSION,
[request_id],
)
# Helper functions
def _create_key(
self,
token_ids: list[int],
start: int = 0,
end: int = 0,
request_id: str | None = None,
) -> IPCCacheEngineKey:
"""Convert token IDs to an IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=self.worker_id,
token_ids=tuple(token_ids),
start=start,
end=end,
request_id=request_id,
tp_size=self.tp_size,
)
def _create_hash_key(
self, chunk_hash: bytes, request_id: str | None = None
) -> IPCCacheEngineKey:
"""Create a hash-mode IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=None,
chunk_hash=chunk_hash,
request_id=request_id,
tp_size=self.tp_size,
)
class LMCacheMPWorkerAdapter:
def __init__(
self,
server_url: str,
context: zmq.Context,
model_name: str,
vllm_block_size: int,
parallel_strategy: ParallelStrategy,
):
self.mq_client = MessageQueueClient(server_url, context)
# Instance id for GPU worker
self.instance_id = os.getpid()
# Registered kv caches from vLLM
self.kv_caches: dict[str, torch.Tensor] = {}
# Request futures
# request_id -> (future, other merged requests)
self.store_futures: dict[
str, tuple[MessagingFuture[StoreResult], list[str]]
] = {}
self.retrieve_futures: dict[
str, tuple[MessagingFuture[RetrieveResult], list[str]]
] = {}
# The store requests that have finished execution in LMCache
self.finished_stores: set[str] = set()
# The finished request ids that are passed via vLLM and also
# have corresponding store requests submitted to LMCache before
self.previously_finished: set[str] = set()
self.model_name = model_name
self.parallel_strategy = parallel_strategy
# Read chunk size from lmcache
chunk_size = get_lmcache_chunk_size(self.mq_client)
assert chunk_size % vllm_block_size == 0, (
"LMCache chunk size should be a multiple of vLLM block size"
)
self.blocks_in_chunk = chunk_size // vllm_block_size
@property
def world_size(self) -> int:
"""The world size."""
return self.parallel_strategy.kv_world_size
@property
def worker_id(self) -> int:
"""The worker id."""
return self.parallel_strategy.kv_worker_id
@property
def use_mla(self) -> bool:
"""Whether to use MLA."""
return self.parallel_strategy.use_mla
@property
def is_first_rank_of_pp_group(self) -> bool:
"""Is the first rank of the pipeline parallel group."""
return (
self.parallel_strategy.actual_worker_id % self.parallel_strategy.tp_size
== 0
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the kv caches with LMCache server.
Args:
kv_caches: A dict of kv caches to register. The keys are the
layer names and the values are the corresponding tensors.
"""
# Register kv cache and send the request
self.kv_caches = kv_caches
logger.info("Registering kv caches")
future = send_lmcache_request(
self.mq_client,
RequestType.REGISTER_KV_CACHE,
[self.instance_id, wrap_kv_caches(kv_caches)],
)
future.result()
@_lmcache_nvtx_annotate
def submit_store_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
):
"""Submit a KV cache store request to LMCache.
Args:
request_id: The ID of the request
op: The LoadStoreOp describing the store operation.
event: The CUDA event that is recorded after the current
model inference step
"""
if op.block_hashes is not None:
# Hash mode
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
]
else:
# Token mode
assert op.token_ids is not None
keys = [
self._create_key(op.token_ids, op.start, op.end, request_id=request_id)
]
future = send_lmcache_request(
self.mq_client,
RequestType.STORE,
[keys, self.instance_id, op.block_ids, event.ipc_handle()],
).to_cuda_future()
self.store_futures[request_id] = (future, [])
@_lmcache_nvtx_annotate
def submit_retrieve_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
):
"""Submit a KV cache retrieve request to LMCache.
Args:
request_id: The ID of the request
op: The LoadStoreOp describing the retrieve operation.
event: The CUDA event that is recorded after the current
model inference step
"""
if op.block_hashes is not None:
# Hash mode
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
]
else:
# Token mode
assert op.token_ids is not None
keys = [
self._create_key(op.token_ids, op.start, op.end, request_id=request_id)
]
future = send_lmcache_request(
self.mq_client,
RequestType.RETRIEVE,
[keys, self.instance_id, op.block_ids, event.ipc_handle()],
).to_cuda_future()
self.retrieve_futures[request_id] = (future, [])
@_lmcache_nvtx_annotate
def batched_submit_store_requests(
self,
request_ids: list[str],
ops: list[LoadStoreOp],
event: torch.cuda.Event,
):
"""Submit a batched store request to LMCache.
Args:
request_ids: The IDs of the requests
ops: The LoadStoreOps describing the store operations. Should have
the same length as request_ids
event: The CUDA event that is recorded after the current
model inference step
"""
all_keys: list[IPCCacheEngineKey] = []
block_ids: list[int] = []
for request_id, op in zip(request_ids, ops, strict=False):
if op.block_hashes is not None:
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id)
for ch in chunk_hashes
]
all_keys.extend(keys)
else:
assert op.token_ids is not None
all_keys.append(
self._create_key(
op.token_ids, op.start, op.end, request_id=request_id
)
)
block_ids.extend(op.block_ids)
future = send_lmcache_request(
self.mq_client,
RequestType.STORE,
[
all_keys,
self.instance_id,
block_ids,
event.ipc_handle(),
],
).to_cuda_future()
self.store_futures[request_ids[0]] = (future, list(request_ids[1:]))
@_lmcache_nvtx_annotate
def batched_submit_retrieve_requests(
self,
request_ids: list[str],
ops: list[LoadStoreOp],
event: torch.cuda.Event,
):
"""Submit a batched retrieve request to LMCache.
Args:
request_ids: The IDs of the requests
ops: The LoadStoreOps describing the retrieve operations. Should have
the same length as request_ids
event: The CUDA event that is recorded after the current
model inference step
"""
all_keys: list[IPCCacheEngineKey] = []
block_ids: list[int] = []
for request_id, op in zip(request_ids, ops, strict=False):
if op.block_hashes is not None:
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id)
for ch in chunk_hashes
]
all_keys.extend(keys)
else:
assert op.token_ids is not None
all_keys.append(
self._create_key(
op.token_ids, op.start, op.end, request_id=request_id
)
)
block_ids.extend(op.block_ids)
future = send_lmcache_request(
self.mq_client,
RequestType.RETRIEVE,
[
all_keys,
self.instance_id,
block_ids,
event.ipc_handle(),
],
).to_cuda_future()
self.retrieve_futures[request_ids[0]] = (future, list(request_ids[1:]))
@_lmcache_nvtx_annotate
def get_finished(
self, finished_req_ids_from_engine: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Check and get the finished store and retrieve requests.
Args:
finished_req_ids_from_engine: the set of request ids that are
reported as finished from the vLLM engine side.
Returns:
A tuple of two sets:
- The first set contains the finished store request ids. The returned
store request ids MUST be seen before in the
`finished_req_ids_from_engine`.
- The second set contains the finished retrieve request ids.
Notes:
When enabling async scheduling in vLLM, the same request ID may appear
multiple times in `finished_req_ids_from_engine`. The adapter should
take care of deduplicating the request IDs and only return the request
IDs that have not been returned before.
"""
finished_stores = set()
finished_retrieves = set()
for request_id, (s_future, other_reqs) in self.store_futures.items():
if not s_future.query():
continue
s_result = s_future.result()
finished_stores.add(request_id)
finished_stores.update(other_reqs)
if not s_result:
# TODO: add error handling here
logger.error(
"Something went wrong when processing the "
"store request for request_id=%s",
request_id,
)
for request_id, (r_future, other_reqs) in self.retrieve_futures.items():
if not r_future.query():
continue
r_result = r_future.result()
finished_retrieves.add(request_id)
finished_retrieves.update(other_reqs)
if not all(r_result):
# TODO: add error handing here
logger.error(
"Something went wrong when processing the "
"retrieve request for request_id=%s, result=%s",
request_id,
r_result,
)
# Remove the finished requests from the tracking dicts
for request_id in finished_stores:
self.store_futures.pop(request_id, None)
for request_id in finished_retrieves:
self.retrieve_futures.pop(request_id, None)
# Update the internal states
self.finished_stores.update(finished_stores)
ret_stores = set()
for req_id in finished_req_ids_from_engine:
if req_id in self.finished_stores or req_id in self.store_futures:
self.previously_finished.add(req_id)
else:
ret_stores.add(req_id)
# Calculate the final finished stores
ret_stores.update(self._update_and_get_finished_store())
return ret_stores, finished_retrieves
def num_blocks_per_chunk(self) -> int:
"""Returns:
The number of vllm blocks in a LMCache data chunk
"""
return self.blocks_in_chunk
def shutdown(self):
"""Shutdown the LMCache MP worker adapter."""
logger.info("Unregistering kv caches")
send_lmcache_request(
self.mq_client, RequestType.UNREGISTER_KV_CACHE, [self.instance_id]
).result()
self.mq_client.close()
# Helper functions
def _update_and_get_finished_store(
self,
) -> set[str]:
"""Converge the internal states about finished stores
and returns the 'safe finished store request ids' back
"""
safe_finished_s = self.finished_stores.intersection(self.previously_finished)
self.finished_stores.difference_update(self.previously_finished)
self.previously_finished.difference_update(safe_finished_s)
return safe_finished_s
def _create_key(
self,
token_ids: list[int],
start: int = 0,
end: int = 0,
request_id: str | None = None,
) -> IPCCacheEngineKey:
"""Convert token IDs to an IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=self.worker_id,
token_ids=tuple(token_ids),
start=start,
end=end,
request_id=request_id,
)
def _create_hash_key(
self, chunk_hash: bytes, request_id: str | None = None
) -> IPCCacheEngineKey:
"""Create a hash-mode IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=self.worker_id,
chunk_hash=chunk_hash,
request_id=request_id,
)
@@ -1,134 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Standard
import os
import threading
from typing import TYPE_CHECKING, Union
import torch
from lmcache.logging import init_logger
from lmcache.v1.config import LMCacheEngineConfig as V1Config
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.multimodal.inputs import PlaceholderRange
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.request import Request
logger = init_logger(__name__)
ENGINE_NAME = "vllm-instance"
# Thread-safe singleton storage
_config_instance: V1Config | None = None
_config_lock = threading.Lock()
def lmcache_get_or_create_config() -> V1Config:
"""Get the LMCache configuration from the environment variable
`LMCACHE_CONFIG_FILE`. If the environment variable is not set, this
function will return the default configuration.
This function is thread-safe and implements singleton pattern,
ensuring the configuration is loaded only once.
"""
global _config_instance
# Double-checked locking for thread-safe singleton
if _config_instance is None:
with _config_lock:
if _config_instance is None: # Check again within lock
LMCacheEngineConfig = V1Config # type: ignore[assignment]
if "LMCACHE_CONFIG_FILE" not in os.environ:
logger.warning(
"No LMCache configuration file is set. Trying to read"
" configurations from the environment variables."
)
logger.warning(
"You can set the configuration file through "
"the environment variable: LMCACHE_CONFIG_FILE"
)
_config_instance = LMCacheEngineConfig.from_env()
else:
config_file = os.environ["LMCACHE_CONFIG_FILE"]
logger.info("Loading LMCache config file %s", config_file)
_config_instance = LMCacheEngineConfig.from_file(config_file)
# Update config from environment variables
_config_instance.update_config_from_env()
return _config_instance
def hex_hash_to_int16(s: str) -> int:
"""Convert a hex hash string to a 16-bit integer."""
return int(s, 16) & 0xFFFF
def apply_mm_hashes_to_token_ids(
token_ids: torch.Tensor,
mm_hashes: list[str],
mm_positions: list["PlaceholderRange"],
) -> torch.Tensor:
"""Overwrite token_ids in-place for multimodal placeholders using
efficient slice assignments.
"""
n = token_ids.size(0)
for hash_str, placeholder in zip(mm_hashes, mm_positions):
start, length = placeholder.offset, placeholder.length
if start >= n:
continue
end = min(start + length, n)
token_ids[start:end] = hex_hash_to_int16(hash_str)
return token_ids
def mla_enabled(model_config: "ModelConfig") -> bool:
return (
hasattr(model_config, "use_mla")
and isinstance(model_config.use_mla, bool)
and model_config.use_mla
)
def extract_mm_features(
request: Union["Request", "NewRequestData"], modify: bool = False
) -> tuple[list[str], list["PlaceholderRange"]]:
"""Normalize multimodal information from a Request into parallel lists.
This helper reads either:
1) `request.mm_features` (objects each exposing `.identifier` and
`.mm_position`), or
2) legacy fields `request.mm_hashes` and `request.mm_positions`.
It returns two equally sized lists: the multimodal hash identifiers and
their corresponding positions. If the request contains no multimodal info,
it returns `([], [])`.
Args:
request (Request): The source object.
modify (bool):
Controls copy semantics for the legacy-path return values.
- If True and legacy fields are used, shallow-copies are returned so
the caller can mutate the lists without affecting `request`.
- If False, the original legacy sequences are returned as-is
(zero-copy); treat them as read-only.
Returns:
tuple[list[str], list[PlaceholderRange]]: (`mm_hashes`, `mm_positions`).
May be `([], [])` when no multimodal data is present.
"""
if getattr(request, "mm_features", None):
mm_hashes, mm_positions = zip(
*((f.identifier, f.mm_position) for f in request.mm_features)
)
return (list(mm_hashes), list(mm_positions))
elif getattr(request, "mm_hashes", None):
if modify:
return (
request.mm_hashes.copy(), # type: ignore
request.mm_positions.copy(), # type: ignore
)
else:
return (request.mm_hashes, request.mm_positions) # type: ignore
else:
return ([], [])
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,178 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import socket
import threading
import time
from dataclasses import dataclass
import uvicorn
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from vllm.distributed.kv_transfer.kv_connector.utils import EngineId
from vllm.logger import init_logger
WorkerAddr = str
logger = init_logger(__name__)
class RegisterWorkerPayload(BaseModel):
engine_id: EngineId
dp_rank: int
tp_rank: int
pp_rank: int
addr: WorkerAddr
@dataclass
class EngineEntry:
engine_id: EngineId
# {tp_rank: {pp_rank: worker_addr}}
worker_addr: dict[int, dict[int, WorkerAddr]]
class MooncakeBootstrapServer:
"""A centralized registry for prefiller connection info (IP, port, ranks)."""
def __init__(self, host: str, port: int):
self.workers: dict[int, EngineEntry] = {}
self.host = host
self.port = port
self.app = FastAPI()
self._register_routes()
self.server_thread: threading.Thread | None = None
self.server: uvicorn.Server | None = None
self._socket: socket.socket | None = None
self._startup_error: BaseException | None = None
def __del__(self):
self.shutdown()
def _register_routes(self):
# All methods are async. No need to use lock to protect data.
self.app.post("/register")(self.register_worker)
self.app.get("/query", response_model=dict[int, EngineEntry])(self.query)
def start(self, *, timeout: float = 30.0):
if self.server_thread:
if (
self.server_thread.is_alive()
and self.server is not None
and self.server.started
and not self.server.should_exit
and self._socket is not None
and self._socket.fileno() >= 0
):
return
raise RuntimeError(
"Mooncake bootstrap server is still starting or stopping"
)
try:
# Bind in the caller so errors propagate, and retain the listener
# when handing it to Uvicorn, including for automatically chosen ports.
family = socket.AF_INET6 if ":" in self.host else socket.AF_INET
self._socket = socket.create_server((self.host, self.port), family=family)
self.port = self._socket.getsockname()[1]
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
server = self.server = uvicorn.Server(config=config)
listener = self._socket
self._startup_error = None
def run():
try:
server.run(sockets=[listener])
except BaseException as exc:
self._startup_error = exc
self.server_thread = threading.Thread(
target=run, name="mooncake_bootstrap_server", daemon=True
)
self.server_thread.start()
deadline = time.monotonic() + timeout
while not server.started:
if not self.server_thread.is_alive():
raise RuntimeError(
"Mooncake bootstrap server exited during startup"
) from self._startup_error
if time.monotonic() >= deadline:
raise TimeoutError(
"Mooncake bootstrap server did not start in time"
)
time.sleep(0.01)
except BaseException:
self.shutdown()
raise
logger.info("Mooncake Bootstrap Server started at %s:%d", self.host, self.port)
def shutdown(self):
was_started = self.server is not None and self.server.started
if self.server is not None:
self.server.should_exit = True
if self.server_thread is not None and self.server_thread.ident is not None:
self.server_thread.join(timeout=5)
if self._socket is not None:
self._socket.close()
self._socket = None
if self.server_thread is not None and self.server_thread.is_alive():
logger.warning("Mooncake bootstrap server did not stop in time")
else:
self.server_thread = None
self.server = None
if was_started:
logger.info("Mooncake Bootstrap Server stopped.")
async def register_worker(self, payload: RegisterWorkerPayload):
"""Handles registration of a prefiller worker."""
if payload.dp_rank not in self.workers:
self.workers[payload.dp_rank] = EngineEntry(
engine_id=payload.engine_id,
worker_addr={},
)
dp_entry = self.workers[payload.dp_rank]
if dp_entry.engine_id != payload.engine_id:
raise HTTPException(
status_code=400,
detail=(
f"Engine ID mismatch for dp_rank={payload.dp_rank}: "
f"expected {dp_entry.engine_id}, got {payload.engine_id}"
),
)
if payload.tp_rank not in dp_entry.worker_addr:
dp_entry.worker_addr[payload.tp_rank] = {}
tp_entry = dp_entry.worker_addr[payload.tp_rank]
existing = tp_entry.get(payload.pp_rank)
if existing is not None:
# A client timeout can fire after the server recorded the
# registration so an identical retry must not be an error.
if existing == payload.addr:
return {"status": "ok"}
raise HTTPException(
status_code=400,
detail=(
f"Worker with dp_rank={payload.dp_rank}, "
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
f"is already registered at "
f"{existing}, "
f"but still want to register at {payload.addr}"
),
)
tp_entry[payload.pp_rank] = payload.addr
logger.debug(
"Registered worker: engine_id=%s, dp_rank=%d, tp_rank=%d, pp_rank=%d at %s",
payload.engine_id,
payload.dp_rank,
payload.tp_rank,
payload.pp_rank,
payload.addr,
)
return {"status": "ok"}
async def query(self) -> dict[int, EngineEntry]:
return self.workers
@@ -1,46 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake requester config helpers."""
from collections.abc import Mapping
from typing import Any
import vllm.envs as envs
from vllm.logger import init_logger
logger = init_logger(__name__)
def normalize_string_override(value: Any) -> str | None:
if not isinstance(value, str):
return None
normalized = value.strip()
return normalized or None
def get_requester_local_hostname(local_ip: str) -> str:
override = normalize_string_override(envs.MOONCAKE_REQUESTER_LOCAL_HOSTNAME)
if override is not None:
return override
return local_ip
def get_configured_preferred_segment(
extra_config: Mapping[str, Any],
) -> str | None:
preferred_segment = normalize_string_override(extra_config.get("preferred_segment"))
if preferred_segment is not None:
return preferred_segment
if extra_config.get("preferred_segment") is not None:
raise ValueError(
"Mooncake preferred_segment override must be a non-empty string"
)
env_value = normalize_string_override(envs.MOONCAKE_PREFERRED_SEGMENT)
if env_value is not None:
logger.info(
"Mooncake preferred_segment from MOONCAKE_PREFERRED_SEGMENT: %s",
env_value,
)
return env_value
return None
@@ -1,146 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stats container for the Mooncake connector."""
import threading
from dataclasses import dataclass
from typing import Any
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorStats,
)
# TODO(mooncake-stats): add MooncakePromMetrics (mirror NixlPromMetrics)
# and wire it via MooncakeConnector.build_prom_metrics in a follow-up PR.
@dataclass
class MooncakeKVConnectorStats(KVConnectorStats):
"""Container for Mooncake KV transfer performance metrics.
`_lock` serializes record_* against clone_and_reset so each row's
appends are atomic and column lengths stay aligned. Writers run on
the sender pool / receiver loop / sender loop; reader runs on the
main worker thread.
"""
def __post_init__(self):
self._lock = threading.Lock()
if not self.data:
self.reset()
# threading.Lock is not picklable; strip it from the wire form and
# rebuild a fresh per-process lock on the receiver side.
def __getstate__(self) -> dict[str, Any]:
state = self.__dict__.copy()
state.pop("_lock", None)
return state
def __setstate__(self, state: dict[str, Any]) -> None:
self.__dict__.update(state)
self._lock = threading.Lock()
def reset(self):
self.data: dict[str, list[float | int]] = {
"transfer_duration": [],
"bytes_transferred": [],
"num_descriptors": [],
"num_failed_transfers": [],
"num_failed_recvs": [],
"num_kv_expired_reqs": [],
}
def record_transfer(self, duration_s: float, total_bytes: int, num_descs: int):
with self._lock:
self.data["transfer_duration"].append(duration_s)
self.data["bytes_transferred"].append(total_bytes)
self.data["num_descriptors"].append(num_descs)
# Failure counters store a list of 1s so a future Prom counter can iterate
# with .inc(list_item), mirroring NIXL's NixlPromMetrics.observe.
def record_failed_transfer(self):
with self._lock:
self.data["num_failed_transfers"].append(1)
def record_failed_recv(self):
with self._lock:
self.data["num_failed_recvs"].append(1)
def record_kv_expired_req(self):
with self._lock:
self.data["num_kv_expired_reqs"].append(1)
def clone_and_reset(self) -> "MooncakeKVConnectorStats":
# Copy lists under the lock for length alignment; return a fresh
# instance so the snapshot has its own _lock.
with self._lock:
snapshot_data: dict[str, list[float | int]] = {
k: list(v) for k, v in self.data.items()
}
self.reset()
return MooncakeKVConnectorStats(data=snapshot_data)
def is_empty(self) -> bool:
return (
self.num_successful_transfers == 0
and len(self.data["num_failed_transfers"]) == 0
and len(self.data["num_failed_recvs"]) == 0
and len(self.data["num_kv_expired_reqs"]) == 0
)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
accumulator = self.data[k]
assert isinstance(accumulator, list)
accumulator.extend(v)
return self
def reduce(self) -> dict[str, int | float]:
num_failed_transfers = len(self.data["num_failed_transfers"])
num_failed_recvs = len(self.data["num_failed_recvs"])
num_kv_expired_reqs = len(self.data["num_kv_expired_reqs"])
if self.num_successful_transfers == 0:
return {
"Num successful transfers": 0,
"Avg xfer time (ms)": 0,
"P90 xfer time (ms)": 0,
"Avg MB per transfer": 0,
"Throughput (MB/s)": 0,
"Avg number of descriptors": 0,
"Num failed transfers": num_failed_transfers,
"Num failed recvs": num_failed_recvs,
"Num KV expired reqs": num_kv_expired_reqs,
}
xfer_time = np.asarray(self.data["transfer_duration"])
mb = np.asarray(self.data["bytes_transferred"]) / 2**20
descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32)
n = len(descs)
assert n == self.num_successful_transfers
total_mb = mb.sum()
avg_mb = total_mb / n
total_time_seconds = xfer_time.sum()
throughput_mb_s = (
(total_mb / total_time_seconds).item() if total_time_seconds > 0 else 0.0
)
return {
"Num successful transfers": n,
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
"Avg MB per transfer": round(avg_mb.item(), 3),
"Throughput (MB/s)": round(throughput_mb_s, 3),
"Avg number of descriptors": round(descs.mean().item(), 1),
"Num failed transfers": num_failed_transfers,
"Num failed recvs": num_failed_recvs,
"Num KV expired reqs": num_kv_expired_reqs,
}
@property
def num_successful_transfers(self) -> int:
return len(self.data["transfer_duration"])
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,436 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from vllm-project/vllm-ascend
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""MooncakeStoreConnector - KV cache connector using MooncakeDistributedStore.
Unlike MooncakeConnector which does direct P2P transfer, this connector
uses MooncakeDistributedStore as a shared KV cache pool. Both producer
and consumer instances read/write KV to/from the store independently,
enabling prefix caching via hash-based deduplication.
"""
from collections import Counter
from collections.abc import Iterable, Sequence
from contextlib import AbstractContextManager
from typing import Any
import torch
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_events import (
BlockStored,
KVCacheEvent,
KVConnectorKVEvents,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
KVConnectorWorkerMetadata,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
from .data import MooncakeStoreConnectorMetadata
from .metrics import MooncakeStoreConnectorStats, MooncakeStorePromMetrics
from .scheduler import MooncakeStoreScheduler
from .worker import MooncakeStoreWorker
logger = init_logger(__name__)
class MooncakeStoreKVEvents(KVConnectorKVEvents):
"""KV event aggregation for MooncakeStoreConnector."""
def __init__(
self,
num_workers: int,
group_tp_replication_factors: Sequence[int] = (1,),
) -> None:
if num_workers <= 0:
raise ValueError("num_workers must be greater than zero.")
if any(factor <= 0 for factor in group_tp_replication_factors):
raise ValueError("TP replication factors must be greater than zero.")
self._event_counter: Counter[KVCacheEvent] = Counter()
self._num_workers = num_workers
self._group_tp_replication_factors = tuple(group_tp_replication_factors)
def add_events(self, events: list[KVCacheEvent]) -> None:
if not isinstance(events, list):
raise TypeError("events must be a list of KVCacheEvent.")
self._event_counter.update(events)
def _replication_factor(self, event: KVCacheEvent) -> int:
if not isinstance(event, BlockStored) or event.group_idx is None:
return 1
return self._group_tp_replication_factors[event.group_idx]
def _is_common_event(self, event: KVCacheEvent, count: int) -> bool:
return count * self._replication_factor(event) >= self._num_workers
def aggregate(self) -> "MooncakeStoreKVEvents":
common_events = self.pop_common_events()
self._event_counter.clear()
self._event_counter.update(common_events)
self._num_workers = 1
return self
def pop_common_events(self) -> list[KVCacheEvent]:
common_events = [
event
for event, count in self._event_counter.items()
if self._is_common_event(event, count)
]
for event in common_events:
del self._event_counter[event]
return common_events
def has_events(self) -> bool:
return bool(self._event_counter)
def increment_workers(self, count: int = 1) -> None:
if count <= 0:
raise ValueError("count must be positive.")
self._num_workers += count
def get_all_events(self) -> list[KVCacheEvent]:
return list(self._event_counter.elements())
def get_number_of_workers(self) -> int:
return self._num_workers
def clear_events(self) -> None:
self._event_counter.clear()
self._num_workers = 1
def __repr__(self) -> str:
return f"<MooncakeStoreKVEvents events={self.get_all_events()}>"
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
"""KV connector using MooncakeDistributedStore as shared KV pool."""
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
return False
@staticmethod
def _validate_kv_cache_config(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> None:
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
unsupported: list[str] = []
store_group_ids = kv_cache_config.prefix_cacheable_group_ids
if not store_group_ids:
raise ValueError(
"MooncakeStore requires at least one prefix-cacheable KV cache group"
)
for group_id in store_group_ids:
spec = kv_cache_config.kv_cache_groups[group_id].kv_cache_spec
if isinstance(spec, CrossAttentionSpec):
unsupported.append(f"group {group_id}: CrossAttentionSpec")
if isinstance(spec, MambaSpec) and spec.mamba_cache_mode != "align":
unsupported.append(
f"group {group_id}: mamba_cache_mode="
f"{spec.mamba_cache_mode!r} != 'align'"
)
pcp = vllm_config.parallel_config.prefill_context_parallel_size
if len(store_group_ids) > 1 and pcp > 1:
unsupported.append(f"PCP > 1 (pcp={pcp}) with hybrid attention")
if unsupported:
raise ValueError(
"MooncakeStoreConnector does not support: " + "; ".join(unsupported)
)
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig | None = None,
):
super().__init__(
vllm_config=vllm_config,
role=role,
kv_cache_config=kv_cache_config, # type: ignore[arg-type]
)
assert vllm_config.kv_transfer_config is not None
assert kv_cache_config is not None, "kv_cache_config is required"
self.kv_role = vllm_config.kv_transfer_config.kv_role
extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
save_decode_cache = extra_config.get("save_decode_cache", False)
# Capacity-only: contributes its segment to the store pool but transfers
# no KV, so the KV-cache-shape invariants below cannot be reached.
self._capacity_only = (
self.kv_role == "kv_consumer"
and not extra_config.get("enable_lookup", True)
and not save_decode_cache
)
if not self._capacity_only:
self._validate_kv_cache_config(vllm_config, kv_cache_config)
self._kv_cache_config = kv_cache_config
self._kv_cache_events: MooncakeStoreKVEvents | None = None
self.connector_scheduler: MooncakeStoreScheduler | None = None
self.connector_worker: MooncakeStoreWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = MooncakeStoreScheduler(
vllm_config, kv_cache_config
)
else:
self.connector_worker = MooncakeStoreWorker(vllm_config, kv_cache_config)
def shutdown(self):
"""Release connector resources on teardown.
Closes the worker's MooncakeDistributedStore handle so its
TransferEngine and RDMA registrations are released. Invoked from the
engine's explicit shutdown path and as a backstop from ``__del__``;
a no-op on the scheduler role, which holds no store handle.
"""
worker = getattr(self, "connector_worker", None)
if worker is not None:
worker.close()
def __del__(self):
self.shutdown()
# ============================================================
# Scheduler-side methods
# ============================================================
def get_num_new_matched_tokens(
self,
request: Request,
num_computed_tokens: int,
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
):
assert self.connector_scheduler is not None
return self.connector_scheduler.update_state_after_alloc(
request, blocks, num_external_tokens
)
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_pending_push_work()
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
assert self.connector_worker is not None
return self.connector_worker.build_connector_worker_meta()
def request_finished(
self,
request: Request,
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
return self.request_finished_all_groups(request, (block_ids,))
def request_finished_all_groups(
self,
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
# An in-flight store job holds its own reference on the blocks it reads,
# so a finishing request never has to defer freeing them.
return False, None
def register_finished_partial_tail(
self,
request: Request,
block_ids: tuple[list[int], ...],
partial_tail_offloads: list[tuple[int, int, int]],
) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.register_finished_partial_tail(
request, block_ids, partial_tail_offloads
)
def reset_cache(self) -> bool | None:
"""Reset the external Mooncake store on prefix-cache reset.
Drains the worker send queue, then runs ``remove_all`` on the
Mooncake master. Caller must first pause generation (e.g.
``pause_generation``) so no new puts are enqueued during drain.
Returns True on ack, False on failure, None for the worker role.
"""
if self.role == KVConnectorRole.SCHEDULER:
assert self.connector_scheduler is not None
# Clear local references to keys we're about to wipe.
self.connector_scheduler.load_specs.clear()
self._kv_cache_events = None
return self.connector_scheduler.reset_store()
return None
def update_connector_output(self, connector_output: KVConnectorOutput):
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
kv_cache_events = connector_output.kv_cache_events
if not kv_cache_events or not isinstance(
kv_cache_events, MooncakeStoreKVEvents
):
return
if self._kv_cache_events is None:
self._kv_cache_events = kv_cache_events
else:
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
def take_events(self) -> Iterable[KVCacheEvent]:
if self._kv_cache_events is not None:
events = self._kv_cache_events.pop_common_events()
if not self._kv_cache_events.has_events():
self._kv_cache_events = None
yield from events
# ============================================================
# Worker-side methods
# ============================================================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a context manager for the custom MemPool, or None.
Called by the Worker before ``initialize_kv_cache`` so that KV
cache is allocated from the Mooncake-managed pool when
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
"""
if self.connector_worker is None:
return None
return self.connector_worker.get_mem_pool_context()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
self.connector_worker.start_load_kv(metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
# No layerwise support - no-op
return
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
# No layerwise support - no-op
return
def wait_for_save(self):
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
self.connector_worker.wait_for_save(metadata)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_finished(finished_req_ids, metadata)
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_transfer_results(finished_req_ids, metadata)
def get_block_ids_with_load_errors(self) -> set[int]:
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
def get_kv_connector_kv_cache_events(
self,
) -> MooncakeStoreKVEvents | None:
assert self.connector_worker is not None
if (
not self.connector_worker.enable_kv_events
or self.connector_worker.kv_send_thread is None
):
return None
events = self.connector_worker.get_kv_events()
# Empty containers still count this worker toward the poll's quorum.
kv_events = MooncakeStoreKVEvents(
num_workers=1,
group_tp_replication_factors=(
self.connector_worker.group_tp_replication_factors
),
)
kv_events.add_events(events)
return kv_events
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
MooncakeStoreConnectorStats(data=data)
if data is not None
else MooncakeStoreConnectorStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return MooncakeStorePromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
@@ -1,515 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
from collections.abc import Sequence
from dataclasses import replace
from typing import TYPE_CHECKING, NamedTuple, cast
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
chunk_hashes_for_block_size,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_coordinator import replay_boundaries
from vllm.v1.core.kv_cache_utils import (
BlockHash,
KVCacheBlock,
eagle_proof_margin,
resolve_dcp_kv_cache_spec,
)
from vllm.v1.core.single_type_kv_cache_manager import (
SingleTypeKVCacheManager,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
if TYPE_CHECKING:
from vllm.config import VllmConfig
class StoreSpecGroup(NamedTuple):
spec: KVCacheSpec
group_ids: list[int]
manager_cls: type[SingleTypeKVCacheManager]
use_eagle: bool
class ExternalCachedBlockPool:
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
def __init__(
self,
hash_block_size: int,
exists: set[tuple[int, bytes]] | None = None,
) -> None:
# ``exists=None`` is used on the recv side where hit_length is already
# determined and we just want each spec's manager to apply its own mask.
self._exists = exists
self.hash_block_size = hash_block_size
self.null_block = KVCacheBlock(block_id=0)
# Dummy ID 1 for present block for duck-typing.
self._present_block = KVCacheBlock(block_id=1)
def get_cached_block(
self,
block_hash: BlockHash,
group_ids: list[int],
) -> list[KVCacheBlock] | None:
# Mirrors BlockPool.get_cached_block: hit only when every group_id
# (groups sharing a spec) has the hash cached.
if self._exists is None:
return [self._present_block] * len(group_ids)
h = bytes(block_hash)
if all((g, h) in self._exists for g in group_ids):
return [self._present_block] * len(group_ids)
return None
def contains(self, group_id: int, block_hash: BlockHash) -> bool:
"""Return whether a group has a loadable key for this hash."""
return (
self._exists is not None and (group_id, bytes(block_hash)) in self._exists
)
class MooncakeStoreCoordinator:
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
``ExternalCachedBlockPool``."""
def __init__(
self,
kv_cache_groups: list[KVCacheGroupSpec],
scheduler_block_size: int,
hash_block_size: int,
use_eagle: bool = False,
retention_interval: int | None = None,
enable_partial_hash_hits: bool = False,
) -> None:
# Mirrors core's resolve_kv_cache_block_sizes: the hash unit only has
# to divide groups that participate in prefix caching. Non-shareable
# scratch groups (e.g. GLM-5.3-Flash's kpool tail) are skipped by
# _verify_and_split_kv_cache_groups and never probed for hits.
assert all(
g.kv_cache_spec.block_size % hash_block_size == 0
for g in kv_cache_groups
if g.kv_cache_spec.prefix_cacheable
), "block_size must be divisible by hash_block_size"
assert scheduler_block_size % hash_block_size == 0, (
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
f"hash_block_size ({hash_block_size})"
)
assert all(
scheduler_block_size % g.kv_cache_spec.block_size == 0
for g in kv_cache_groups
if g.kv_cache_spec.prefix_cacheable
), "scheduler_block_size must be a multiple of each group's block_size"
self.kv_cache_groups = kv_cache_groups
self.mamba_group_ids = {
group_id
for group_id, group in enumerate(kv_cache_groups)
if isinstance(_unwrap_spec(group.kv_cache_spec), MambaSpec)
}
self.hash_block_size = hash_block_size
self.lcm_block_size = scheduler_block_size
# Core's decision: whether hits land on the hash unit, below the block.
self.enable_partial_hash_hits = enable_partial_hash_hits
self.use_eagle = use_eagle
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
self.retention_interval = retention_interval
self._verify_and_split_kv_cache_groups()
@classmethod
def from_kv_cache_config(
cls,
kv_cache_config: KVCacheConfig,
vllm_config: "VllmConfig",
scheduler_block_size: int,
hash_block_size: int,
) -> "MooncakeStoreCoordinator":
"""Build the coordinator the connector's scheduler and workers share.
Specs are DCP-resolved, so a block spans the tokens of one block-table
entry. Partial hits follow core's hit alignment, which checks every KV
cache group and its manager, not only the groups the store transfers;
unset (a config the engine core did not resolve) keeps hits on the
block.
"""
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
spec_config = vllm_config.speculative_config
alignment = kv_cache_config.cache_hit_alignment_tokens
return cls(
[
replace(
group,
kv_cache_spec=resolve_dcp_kv_cache_spec(
group.kv_cache_spec, dcp_world_size
),
)
for group in kv_cache_config.prefix_cacheable_groups
],
scheduler_block_size,
hash_block_size,
use_eagle=spec_config is not None and spec_config.use_eagle_block_drop(),
retention_interval=kv_cache_config.prefix_cache_retention_interval,
enable_partial_hash_hits=(
alignment is not None and alignment < scheduler_block_size
),
)
def align_lookup_length(self, length: int) -> int:
alignment = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
return length // alignment * alignment
def _verify_and_split_kv_cache_groups(self) -> None:
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
dispatches via spec_manager_map (we don't allocate managers).
"""
attention_groups: list[StoreSpecGroup] = []
for i, g in enumerate(self.kv_cache_groups):
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
# kpool tail): per-request scratch, never shareable, so they must
# not participate in hit lookup. Mirrors core's
# KVCacheCoordinator.verify_and_split_kv_cache_groups.
if not g.kv_cache_spec.prefix_cacheable:
continue
spec = _unwrap_spec(g.kv_cache_spec)
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
assert manager_cls is not None, (
f"No manager registered for KVCacheSpec {spec}"
)
for idx, group in enumerate(attention_groups):
if group.spec == spec:
assert manager_cls is group.manager_cls
group.group_ids.append(i)
if g.is_eagle_group and not group.use_eagle:
attention_groups[idx] = group._replace(use_eagle=True)
break
else:
attention_groups.append(
StoreSpecGroup(spec, [i], manager_cls, g.is_eagle_group)
)
# Full attention first (matches upstream convergence ordering).
attention_groups.sort(key=lambda g: not isinstance(g.spec, FullAttentionSpec))
# Conservatively flag all groups when use_eagle is set but none is flagged.
if self.use_eagle and not any(g.use_eagle for g in attention_groups):
attention_groups = [g._replace(use_eagle=True) for g in attention_groups]
self.attention_groups = attention_groups
# Per-group eagle bits. SpecGroup carries use_eagle for the whole
# merged spec group, so the per-group store/lookup masks agree with
# the merged-group hit check, which applies the eagle drop to every
# group sharing the spec.
self.eagle_group_ids = {
gid for g in attention_groups if g.use_eagle for gid in g.group_ids
}
self.eagle_proof_margin_by_group = {
gid: eagle_proof_margin(
group.spec.block_size,
self.hash_block_size,
self.enable_partial_hash_hits
and group.manager_cls.supports_fine_grained_hash_lookup,
)
for group in attention_groups
if group.use_eagle and not isinstance(group.spec, MambaSpec)
for gid in group.group_ids
}
def find_longest_cache_hit(
self,
block_hashes: Sequence[BlockHash],
max_length: int,
cached_block_pool: ExternalCachedBlockPool,
*,
apply_eagle: bool = True,
) -> tuple[tuple[list[bool], ...], int]:
"""Returns ``(load_mask_per_group, hit_length)``. ``mask[g][i]`` is True iff
group ``g`` populates chunk ``i`` locally (e.g. SWA and Mamba tail-only);
recv-side callers skip False slots.
``apply_eagle`` controls whether the per-spec ``use_eagle`` last-block
pop is applied. Lookup callers want it (the drafter requires recomputing
the last block); per-chunk mask callers must not, because ``token_len``
already reflects the eagle-pruned hit length and a second pop would
leave the trailing block unloaded.
"""
blocks_per_group, hit_length = self._find_hit_blocks(
block_hashes, max_length, cached_block_pool, apply_eagle=apply_eagle
)
masks = tuple(
[blk is not cached_block_pool.null_block for blk in blocks]
for blocks in blocks_per_group
)
return masks, hit_length
def load_mask(
self,
block_hashes: Sequence[BlockHash],
token_len: int,
) -> tuple[list[bool], ...]:
"""Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s
spec would populate chunk ``i`` locally at length ``token_len``
(e.g. SWA / Mamba tail-only).
"""
# ``apply_eagle=False`` because ``token_len`` is already the
# eagle-pruned hit length returned by ``client.lookup``. Re-applying
# the pop here would shorten the mask by one extra block; the recv
# thread would then silently skip the trailing chunk yielded by
# ``db.process_tokens`` and leave that block uninitialized in the
# local KV pool.
masks, _ = self.find_longest_cache_hit(
block_hashes,
token_len,
ExternalCachedBlockPool(self.hash_block_size),
apply_eagle=False,
)
return masks
def store_mask(
self,
aligned_token_len: int,
start_token: int = 0,
num_prompt_tokens: int | None = None,
) -> tuple[list[bool] | None, ...]:
"""Per-group store masks for the suffix starting at ``start_token``.
``mask[g][i]`` is True iff the i-th chunk of group ``g`` *after*
``start_token`` should be written to the store so a future cache hit
can consume it. ``None`` is the all-True sentinel for the suffix.
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
so the store retains exactly the blocks the local prefix cache would.
Mamba groups are always all-False: the normal save resolves blocks
positionally from the connector's append-only block-ID snapshot, but
an align-mode mamba block table is not append-only (interior state
blocks are nulled/freed, and speculative decoding relocates the spec
blocks in place), so a positional read may hit a null, freed, or live
speculative-state block and persist wrong bytes under a valid prefix
hash. Mamba state is persisted only through the connector-pinned exact
block hand-off path
(``SchedulerOutput.kv_connector_block_state.boundary_state_offloads``).
"""
return self._reachable_masks(
aligned_token_len,
start_token,
retention_interval=self.retention_interval,
num_prompt_tokens=num_prompt_tokens,
exclude_mamba=True,
)
def lookup_mask(
self,
aligned_token_len: int,
) -> tuple[list[bool] | None, ...]:
"""Per-group lookup masks.
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
looked up as an aligned hit boundary. ``None`` is the all-True
sentinel.
"""
return self._reachable_masks(
aligned_token_len,
0,
retention_interval=None,
num_prompt_tokens=None,
)
def _reachable_masks(
self,
aligned_token_len: int,
start_token: int,
*,
retention_interval: int | None,
num_prompt_tokens: int | None,
exclude_mamba: bool = False,
) -> tuple[list[bool] | None, ...]:
mask_alignment = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
assert aligned_token_len % mask_alignment == 0, (
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
f"{mask_alignment}"
)
reachable_boundaries = (
()
if num_prompt_tokens is None
else replay_boundaries(
num_prompt_tokens,
self.lcm_block_size,
use_eagle=bool(self.eagle_group_ids),
)
)
masks: list[list[bool] | None] = []
for g_idx, g in enumerate(self.kv_cache_groups):
spec = _unwrap_spec(g.kv_cache_spec)
end_chunk = aligned_token_len // spec.block_size
start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size)))
if exclude_mamba and isinstance(spec, MambaSpec):
masks.append([False] * (end_chunk - start_chunk))
continue
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
assert manager_cls is not None
use_eagle = g_idx in self.eagle_group_ids
mask = manager_cls.reachable_block_mask(
start_block=start_chunk,
end_block=end_chunk,
alignment_tokens=self.lcm_block_size,
kv_cache_spec=spec,
use_eagle=use_eagle,
retention_interval=retention_interval,
reachable_boundaries=reachable_boundaries,
# ``spec`` is already DCP-resolved (worker.py applies
# resolve_dcp_kv_cache_spec) and ``end_chunk`` is indexed in
# that scaled block size, so the mask must not scale again.
dcp_world_size=1,
)
if mask is not None:
assert len(mask) == end_chunk - start_chunk
masks.append(mask)
return tuple(masks)
def block_hashes_for_spec(
self, block_hashes: Sequence[BlockHash], spec: KVCacheSpec
) -> Sequence[BlockHash]:
return chunk_hashes_for_block_size(
block_hashes, self.hash_block_size, spec.block_size
)
def _find_hit_blocks(
self,
block_hashes: Sequence[BlockHash],
max_length: int,
cached_block_pool: ExternalCachedBlockPool,
*,
apply_eagle: bool = True,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
"""Mirrors HybridKVCacheCoordinator.find_longest_cache_hit but
dispatches via spec_manager_map (we don't allocate managers).
When ``apply_eagle`` is False, ignore each group's ``use_eagle`` —
used by ``load_mask`` to avoid popping a second block on top of the
one already removed by the lookup.
"""
alignment_tokens = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
if len(self.attention_groups) == 1:
spec, group_ids, manager_cls, group_eagle = self.attention_groups[0]
hit_blocks, hit_length = manager_cls.find_longest_cache_hit(
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
drop_eagle_block=apply_eagle and group_eagle,
alignment_tokens=alignment_tokens,
)
num_groups = len(self.kv_cache_groups)
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
for gid, blks in zip(group_ids, hit_blocks, strict=True):
blocks_by_group[gid] = blks
return tuple(blocks_by_group), hit_length
num_groups = len(self.kv_cache_groups)
hit_length = max_length
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
hit_length_by_group: list[int] = [0] * num_groups
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
self.attention_groups[0].spec, FullAttentionSpec
)
eagle_verified: set[int] = set()
while True:
curr_hit_length = hit_length
for idx, (spec, group_ids, manager_cls, group_eagle) in enumerate(
self.attention_groups
):
first_group_id = group_ids[0]
cached = hit_blocks_by_group[first_group_id]
if isinstance(spec, FullAttentionSpec) and cached is not None:
curr_hit_length = min(
curr_hit_length, hit_length_by_group[first_group_id]
)
continue
drop_eagle_block = (
apply_eagle and group_eagle and idx not in eagle_verified
)
_max_length = curr_hit_length
# No eagle peek margin for a recurrent (Mamba) group: its finder
# never drops a block, so a widened bound would match past the
# attention-verified hit and resume from speculative state (#43559).
if drop_eagle_block and not isinstance(spec, MambaSpec):
eagle_margin = eagle_proof_margin(
spec.block_size,
self.hash_block_size,
self.enable_partial_hash_hits
and manager_cls.supports_fine_grained_hash_lookup,
)
_max_length = min(
curr_hit_length + eagle_margin,
len(block_hashes) * self.hash_block_size,
)
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=_max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
drop_eagle_block=drop_eagle_block,
alignment_tokens=alignment_tokens,
)
if drop_eagle_block:
eagle_verified.add(idx)
elif _new_hit_length < curr_hit_length:
eagle_verified.clear()
curr_hit_length = _new_hit_length
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
hit_blocks_by_group[gid] = blocks
hit_length_by_group[gid] = _new_hit_length
if curr_hit_length >= hit_length:
break
hit_length = curr_hit_length
if is_simple_hybrid:
break
# Truncate full-attention hit_blocks to final converged length;
# other specs already trim themselves inside their hit logic. cdiv keeps
# the partial tail block when hit_length is not block-aligned.
for group in self.attention_groups:
if not isinstance(group.spec, FullAttentionSpec):
continue
num_blocks = cdiv(hit_length, group.spec.block_size)
for group_id in group.group_ids:
full_blks = hit_blocks_by_group[group_id]
assert full_blks is not None
del full_blks[num_blocks:]
hit_length_by_group[group_id] = hit_length
return (
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
hit_length,
)
def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
if isinstance(spec, UniformTypeKVCacheSpecs):
return next(iter(spec.kv_cache_specs.values()))
return spec
@@ -1,898 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from vllm-project/vllm-ascend
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""Data classes for MooncakeStoreConnector."""
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from typing import NamedTuple, cast
import numpy as np
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import is_non_overlapping_and_dense
from vllm.v1.core.kv_cache_utils import (
BlockHash,
BlockHashListWithBlockSize,
)
logger = init_logger(__name__)
class BlobBlockHashes(Sequence[BlockHash]):
"""Lazy view over a flat buffer of fixed-size block hashes to avoid the overhead
of materializing all hashes upfront.
"""
def __init__(self, blob: memoryview, hash_len: int):
self._blob = blob
self._hash_len = hash_len
self._n = len(blob) // hash_len if hash_len else 0
def __len__(self) -> int:
return self._n
def __getitem__(self, idx):
if isinstance(idx, slice):
return [self[i] for i in range(*idx.indices(self._n))]
if idx < 0:
idx += self._n
if not 0 <= idx < self._n:
raise IndexError(idx)
off = idx * self._hash_len
return BlockHash(self._blob[off : off + self._hash_len])
class _CompactChunkHashList(BlockHashListWithBlockSize):
"""View that keys each ``block_size`` chunk by the last constituent
``hash_block_size`` hash instead of concatenating all of them.
The engine chains block hashes (each hash folds in the previous one), so the
final sub-block hash of a chunk already uniquely identifies the whole chunk
and its prefix. Using it keeps a Mooncake key at a single hash digest
regardless of the ``block_size`` / ``hash_block_size`` ratio, instead of
growing the key linearly with it (e.g. 64x for ``block_size=256``,
``hash_block_size=4``).
"""
def __init__(
self,
block_hashes: Sequence[BlockHash],
hash_block_size: int,
target_block_size: int,
):
# Accept any indexable sequence (e.g. the lazy ``BlobBlockHashes``), not
# just ``list``; the base only indexes/sizes it.
assert target_block_size % hash_block_size == 0
self.block_hashes = block_hashes # type: ignore[assignment]
self.scale_factor = target_block_size // hash_block_size
def _get_value_at(self, idx: int) -> BlockHash:
return self.block_hashes[idx * self.scale_factor + self.scale_factor - 1]
def chunk_hashes_for_block_size(
block_hashes: Sequence[BlockHash],
hash_block_size: int,
block_size: int,
) -> Sequence[BlockHash]:
"""Map ``hash_block_size``-granular block hashes to one compact hash per
``block_size`` chunk (the chunk's last sub-hash). Returns ``block_hashes``
unchanged when the two sizes are equal.
"""
if block_size == hash_block_size:
return block_hashes
# Structurally a Sequence[BlockHash] (indexable + sized); the base class
# just isn't declared as one.
return cast(
"Sequence[BlockHash]",
_CompactChunkHashList(block_hashes, hash_block_size, block_size),
)
@dataclass
class KeyMetadata:
"""Metadata for constructing pool keys."""
model_name: str
tp_rank: int
pcp_rank: int
dcp_rank: int
pp_rank: int
group_id: int = 0
# Optional namespace prepended to every key. Lets separate deployments
# share one Mooncake master without colliding on identical block hashes.
# Empty (the default) keeps keys byte-identical to the unprefixed format.
cache_prefix: str = ""
# Complete namespace for an opt-in Store payload format. Empty keeps
# historical keys byte-identical.
store_namespace: str = ""
@dataclass(order=True)
class PoolKey:
"""Key for addressing KV cache blocks in the distributed store."""
key_metadata: KeyMetadata
chunk_hash: str
def __hash__(self):
return hash(
(
self.key_metadata.cache_prefix,
self.key_metadata.model_name,
self.key_metadata.store_namespace,
self.key_metadata.tp_rank,
self.key_metadata.pcp_rank,
self.key_metadata.dcp_rank,
self.key_metadata.pp_rank,
self.key_metadata.group_id,
self.chunk_hash,
)
)
@staticmethod
def build_prefix(
key_metadata: KeyMetadata,
*,
tp_rank: int | None = None,
pcp_rank: int | None = None,
dcp_rank: int | None = None,
pp_rank: int | None = None,
) -> str:
"""Return the stable prefix for a Mooncake pool key."""
prefix = f"{key_metadata.cache_prefix}@" if key_metadata.cache_prefix else ""
return (
f"{prefix}"
f"{key_metadata.model_name}"
f"{key_metadata.store_namespace}"
f"@tp_rank:{key_metadata.tp_rank if tp_rank is None else tp_rank}"
f"@pcp{key_metadata.pcp_rank if pcp_rank is None else pcp_rank}"
f"@dcp{key_metadata.dcp_rank if dcp_rank is None else dcp_rank}"
f"@pp_rank:{key_metadata.pp_rank if pp_rank is None else pp_rank}"
f"@group:{key_metadata.group_id}"
)
@staticmethod
def build_key_string(key_prefix: str, chunk_hash: str) -> str:
return f"{key_prefix}@{chunk_hash}"
def to_string(self) -> str:
return self.build_key_string(
self.build_prefix(self.key_metadata), self.chunk_hash
)
StoreShardId = int
# Physical ranks encoded in a rank-local key: (TP, PCP, DCP, PP).
RankNamespace = tuple[int, int, int, int]
class StoreLayout:
"""Store payload layout for one KV-cache group on one local TP rank."""
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int,
) -> None:
self.metadata = metadata
self.block_size = block_size
self.hash_block_size = hash_block_size
@property
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
"""Store objects contributed or consumed by this local rank."""
raise NotImplementedError
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
raise NotImplementedError
def lookup_key_prefixes(
self,
rank_namespaces: Sequence[RankNamespace],
) -> tuple[str, ...]:
"""Prefixes that must exist for a logical block to be reusable."""
raise NotImplementedError
def register_kv_caches(
self,
kv_caches: Sequence[torch.Tensor],
num_blocks: int,
) -> None:
"""Build descriptors from the local KV cache tensors."""
raise NotImplementedError
def prepare_values(
self,
chunks: Sequence[tuple[int, int]],
block_ids: list[int],
shard_ids: Sequence[StoreShardId],
) -> tuple[list[list[int]], list[list[int]], list[int]]:
"""Build Store multi-buffer descriptors for logical chunks."""
raise NotImplementedError
class RankLocalStoreLayout(StoreLayout):
"""Historical rank-local Store payload layout."""
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int,
) -> None:
super().__init__(metadata, block_size, hash_block_size)
self._key_prefix = PoolKey.build_prefix(metadata)
self.kv_caches_base_addr: list[int] = []
self.block_len: list[int] = []
@property
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
return (0,)
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
assert shard_id == 0
return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex())
def lookup_key_prefixes(
self,
rank_namespaces: Sequence[RankNamespace],
) -> tuple[str, ...]:
return tuple(
PoolKey.build_prefix(
self.metadata,
tp_rank=tp_rank,
pcp_rank=pcp_rank,
dcp_rank=dcp_rank,
pp_rank=pp_rank,
)
for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces
)
def set_kv_caches_base_addr(self, base_addrs: list[int]) -> None:
self.kv_caches_base_addr = base_addrs
def set_block_len(self, block_lens: list[int]) -> None:
self.block_len = block_lens
def register_kv_caches(
self,
kv_caches: Sequence[torch.Tensor],
num_blocks: int,
) -> None:
seen_region_ptrs: set[int] = set()
base_addrs: list[int] = []
block_lens: list[int] = []
for cache in kv_caches:
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
storage_len = storage.nbytes()
if not is_non_overlapping_and_dense(cache[0]):
for head_idx in range(cache.shape[1]):
head_cache = cache[:, head_idx]
assert is_non_overlapping_and_dense(head_cache[0])
region_addr = head_cache.data_ptr()
if region_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(region_addr)
base_addrs.append(region_addr)
block_lens.append(head_cache.stride(0) * head_cache.element_size())
elif cache.stride(0) * cache.element_size() * num_blocks == storage_len:
if storage_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(storage_addr)
base_addrs.append(storage_addr)
block_lens.append(storage_len // num_blocks)
else:
region_addr = cache.data_ptr()
if region_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(region_addr)
base_addrs.append(region_addr)
block_lens.append(cache.stride(0) * cache.element_size())
self.kv_caches_base_addr = base_addrs
self.block_len = block_lens
def prepare_values(
self,
chunks: Sequence[tuple[int, int]],
block_ids: list[int],
shard_ids: Sequence[StoreShardId],
) -> tuple[list[list[int]], list[list[int]], list[int]]:
if not chunks:
return [], [], []
assert len(chunks) == len(shard_ids) and all(
shard_id == 0 for shard_id in shard_ids
)
base = np.asarray(self.kv_caches_base_addr, dtype=np.int64)
length = len(self.block_len)
blen = np.asarray(
[self.block_len[i % length] for i in range(base.shape[0])],
dtype=np.int64,
)
n = len(chunks)
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
assert np.all(spans % self.hash_block_size == 0)
bids = np.fromiter(
(block_ids[i] for i in (starts // self.block_size).tolist()),
dtype=np.int64,
count=n,
)
addrs = base[None, :] + bids[:, None] * blen[None, :]
block_counts = (spans + self.block_size - 1) // self.block_size
sizes = blen[None, :] * block_counts[:, None]
return addrs.tolist(), sizes.tolist(), bids.tolist()
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
length = len(self.block_len)
return (
[
base_addr + block_id * self.block_len[index % length]
for index, base_addr in enumerate(self.kv_caches_base_addr)
],
[
self.block_len[index % length]
for index in range(len(self.kv_caches_base_addr))
],
)
class TPShardedStoreLayout(StoreLayout):
"""Store layout shared by divisible TP sizes."""
store_format: str
@classmethod
def shared_namespace(cls, store_tp_size: int, pp_size: int) -> str:
return (
f"@store_tp:{store_tp_size}@store_pp:{pp_size}"
f"@store_format:{cls.store_format}"
)
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int,
local_tp_size: int,
store_tp_size: int,
tp_rank: int,
num_kv_heads: int,
) -> None:
super().__init__(metadata, block_size, hash_block_size)
self.shards_per_rank = store_tp_size // local_tp_size
first_shard = tp_rank * self.shards_per_rank
self.store_shard_ids = tuple(
range(first_shard, first_shard + self.shards_per_rank)
)
self.store_tp_size = store_tp_size
self.heads_per_store_shard = num_kv_heads // store_tp_size
self.local_num_kv_heads = num_kv_heads // local_tp_size
self._shard_key_prefixes = {
shard_id: PoolKey.build_prefix(metadata, tp_rank=shard_id)
for shard_id in self.store_shard_ids
}
self._shard_addr_bases = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
self._shard_block_strides = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
self._shard_sizes = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
@property
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
return self.store_shard_ids
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
return PoolKey.build_key_string(
self._shard_key_prefixes[shard_id], chunk_hash.hex()
)
def lookup_key_prefixes(
self,
rank_namespaces: Sequence[RankNamespace],
) -> tuple[str, ...]:
pp_ranks = sorted({namespace[3] for namespace in rank_namespaces})
return tuple(
PoolKey.build_prefix(self.metadata, tp_rank=shard_id, pp_rank=pp_rank)
for shard_id in range(self.store_tp_size)
for pp_rank in pp_ranks
)
def _set_segment_templates(
self, templates: Sequence[tuple[list[int], list[int], list[int]]]
) -> None:
self._shard_addr_bases = np.asarray(
[template[0] for template in templates], dtype=np.uint64
)
self._shard_block_strides = np.asarray(
[template[1] for template in templates], dtype=np.uint64
)
self._shard_sizes = np.asarray(
[template[2] for template in templates], dtype=np.uint64
)
def _has_packed_strides(
self, head_stride: int, token_stride: int, content_bytes: int
) -> bool:
raise NotImplementedError
def _shard_segments(
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
) -> Iterable[tuple[int, int]]:
raise NotImplementedError
def register_kv_caches(
self,
kv_caches: Sequence[torch.Tensor],
num_blocks: int,
) -> None:
templates: list[tuple[list[int], list[int], list[int]]] = [
([], [], []) for _ in range(self.shards_per_rank)
]
for cache in kv_caches:
if cache.ndim != 4 or tuple(cache.shape[:3]) != (
num_blocks,
self.local_num_kv_heads,
self.block_size,
):
raise ValueError(
"TP-shared Mooncake store requires packed KV caches with "
"logical shape (num_blocks, local_kv_heads, block_size, content)"
)
element_size = cache.element_size()
block_stride, head_stride, token_stride, content_stride = (
stride * element_size for stride in cache.stride()
)
content_bytes = cache.shape[3] * element_size
if content_stride != element_size or not self._has_packed_strides(
head_stride, token_stride, content_bytes
):
raise ValueError(
"TP-shared Mooncake store requires packed "
f"{self.store_format} KV layout"
)
for local_shard, (addr_bases, block_strides, sizes) in enumerate(templates):
head_start = local_shard * self.heads_per_store_shard
for offset, size in self._shard_segments(
head_start, head_stride, token_stride, content_bytes
):
addr_bases.append(cache.data_ptr() + offset)
block_strides.append(block_stride)
sizes.append(size)
self._set_segment_templates(templates)
def prepare_values(
self,
chunks: Sequence[tuple[int, int]],
block_ids: list[int],
shard_ids: Sequence[StoreShardId],
) -> tuple[list[list[int]], list[list[int]], list[int]]:
if not chunks:
return [], [], []
if len(chunks) != len(shard_ids):
raise ValueError("Each Store chunk must have one shard ID")
count = len(chunks)
starts = np.fromiter(
(chunk[0] for chunk in chunks), dtype=np.int64, count=count
)
ends = np.fromiter((chunk[1] for chunk in chunks), dtype=np.int64, count=count)
if np.any(ends - starts != self.block_size):
raise ValueError("TP-shared Mooncake store requires full KV blocks")
first_shard = self.store_shard_ids[0]
local_shards = np.fromiter(
(shard_id - first_shard for shard_id in shard_ids),
dtype=np.int64,
count=count,
)
if np.any(local_shards < 0) or np.any(local_shards >= self.shards_per_rank):
raise ValueError("Store shard is not owned by this TP rank")
chunk_block_ids = np.fromiter(
(block_ids[index] for index in (starts // self.block_size).tolist()),
dtype=np.uint64,
count=count,
)
addr_bases = self._shard_addr_bases[local_shards]
block_strides = self._shard_block_strides[local_shards]
addrs = addr_bases + chunk_block_ids[:, None] * block_strides
sizes = self._shard_sizes[local_shards]
return addrs.tolist(), sizes.tolist(), chunk_block_ids.tolist()
class LBHNCStoreLayout(TPShardedStoreLayout):
"""Native head-major layout shared by divisible TP sizes."""
store_format = "tp_shared_lbhnc"
def _has_packed_strides(
self, head_stride: int, token_stride: int, content_bytes: int
) -> bool:
return (
token_stride == content_bytes
and head_stride == self.block_size * content_bytes
)
def _shard_segments(
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
) -> Iterable[tuple[int, int]]:
yield head_start * head_stride, self.heads_per_store_shard * head_stride
class LBNHCStoreLayout(TPShardedStoreLayout):
"""Native token-major layout shared by divisible TP sizes."""
store_format = "tp_shared_lbnhc"
def _has_packed_strides(
self, head_stride: int, token_stride: int, content_bytes: int
) -> bool:
return (
head_stride == content_bytes
and token_stride == self.local_num_kv_heads * content_bytes
)
def _shard_segments(
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
) -> Iterable[tuple[int, int]]:
for token_idx in range(self.block_size):
yield (
token_idx * token_stride + head_start * head_stride,
self.heads_per_store_shard * content_bytes,
)
class ChunkedTokenDatabase:
"""Enumerates logical token chunks and their hashes."""
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int | None = None,
store_layout: StoreLayout | None = None,
):
self.metadata = metadata
self.block_size = block_size
self.hash_block_size = hash_block_size or block_size
if self.block_size % self.hash_block_size != 0:
raise ValueError(
f"block_size ({self.block_size}) must be a multiple of "
f"hash_block_size ({self.hash_block_size})"
)
self.store_layout = store_layout or RankLocalStoreLayout(
metadata, block_size, self.hash_block_size
)
@property
def kv_caches_base_addr(self) -> list[int]:
return self._rank_local_layout().kv_caches_base_addr
@property
def block_len(self) -> list[int]:
return self._rank_local_layout().block_len
def _rank_local_layout(self) -> RankLocalStoreLayout:
if not isinstance(self.store_layout, RankLocalStoreLayout):
raise RuntimeError("This operation requires a rank-local Store layout")
return self.store_layout
def key_for(self, chunk_hash: BlockHash) -> str:
return self._rank_local_layout().key_for(0, chunk_hash)
def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]):
self._rank_local_layout().set_kv_caches_base_addr(kv_caches_base_addr)
def set_block_len(self, block_len: list[int]):
self._rank_local_layout().set_block_len(block_len)
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
"""Return addresses and sizes for one physical block slot."""
return self._rank_local_layout().prepare_value_for_block(block_id)
def process_tokens(
self,
token_len: int,
block_hashes: list[BlockHash],
mask_num: int = 0,
*,
chunk_mask: list[bool] | None = None,
put_step: int = 1,
put_step_rank: int = 0,
) -> Iterable[tuple[int, int, BlockHash]]:
"""Process tokens and yield (start_idx, end_idx, block_hash) tuples.
When there are fewer KV heads than TP ranks, chunks are distributed
across TP ranks to avoid duplicate load/store. The assignment keys off
the absolute ``chunk_id`` so a given chunk always lands on the same
rank regardless of where the processed suffix begins.
Args:
token_len: Total number of tokens. Must be hash-block aligned and
covered by ``block_hashes`` when hashes are present.
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
mask_num: Number of tokens to skip from the beginning.
chunk_mask: Optional mask relative to the first chunk after
``mask_num``. False entries are skipped before hash access.
put_step: Stride for distributing chunks across ranks.
put_step_rank: ``chunk_id % put_step`` value this rank stores.
"""
assert put_step > 0
if not block_hashes:
return
assert token_len % self.hash_block_size == 0
assert token_len // self.hash_block_size <= len(block_hashes)
start_chunk = max(0, cdiv(mask_num, self.block_size))
max_chunks = cdiv(token_len, self.block_size)
if chunk_mask is not None:
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
for chunk_id in range(start_chunk, max_chunks):
if chunk_mask is not None and not chunk_mask[chunk_id - start_chunk]:
continue
if chunk_id % put_step != put_step_rank:
continue
start_idx = chunk_id * self.block_size
end_idx = min(start_idx + self.block_size, token_len)
h = block_hashes[end_idx // self.hash_block_size - 1]
yield start_idx, end_idx, h
@dataclass(frozen=True)
class TailKeyBoundary:
"""Hash boundary used to key a group's tail block in the store.
Attributes:
group_id: KV-cache group containing the tail block.
num_tokens: Token boundary whose prefix hash identifies the matched
stored block. The loader uses
``block_hashes[num_tokens // hash_block_size - 1]`` instead of the
hash implied by ``MooncakeLookupResult.hit_length``. This changes
only the load key, not the reusable prefix.
"""
group_id: int
num_tokens: int
@dataclass
class MooncakeLookupResult:
"""Lookup result used to build the subsequent load request.
Attributes:
hit_length: Longest prefix that every KV-cache group can reuse after
their individual cache hits converge.
tail_key_boundaries: Hash boundary used to store each cache group's
tail block when ``hit_length`` does not identify its store key.
There is one entry per group for every nonzero hit.
"""
hit_length: int
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
@dataclass
class LoadSpec:
"""Specification for loading KV cache from external store."""
vllm_cached_tokens: int
kvpool_cached_tokens: int
can_load: bool
token_len: int = 0
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
@dataclass
class RequestTracker:
"""Tracks per-request state across scheduler ticks."""
req_id: str
token_len: int
allocated_block_ids: tuple[list[int], ...]
num_saved_tokens: int = 0
token_ids: list[int] | None = None
has_pending_offload: bool = False
# Snapshot of the prefill range length at tracker creation time.
# For a fresh request this is len(prompt). For a resumed-from-preemption
# request it includes previously-generated tokens, which are re-prefilled.
prefill_end_tokens: int = 0
partial_tail_sent: bool = False
def reset(self) -> None:
self.token_len = 0
self.allocated_block_ids = ()
self.num_saved_tokens = 0
self.token_ids = None
self.has_pending_offload = False
self.prefill_end_tokens = 0
self.partial_tail_sent = False
def update(
self,
new_block_ids: tuple[list[int], ...] | list[int],
) -> None:
# Backward-compat: accept a single list (broadcast to single group).
if isinstance(new_block_ids, list):
new_block_ids = (new_block_ids,)
if len(new_block_ids) != len(self.allocated_block_ids):
raise ValueError(
f"Group count mismatch: tracker has "
f"{len(self.allocated_block_ids)} groups, update has "
f"{len(new_block_ids)}"
)
for existing, new in zip(self.allocated_block_ids, new_block_ids, strict=True):
if new:
existing.extend(new)
class BoundaryPut(NamedTuple):
"""A block stored under the hash of the prefix ending at ``num_tokens``."""
group_id: int
block_id: int
num_tokens: int
@dataclass
class ReqMeta:
"""Per-request metadata for store put/get operations."""
req_id: str
token_len_chunk: int
block_ids: tuple[list[int], ...]
block_hashes: list[BlockHash]
can_save: bool | None = None
load_spec: LoadSpec | None = None
current_event: torch.cuda.Event | None = None
token_ids: list[int] | None = None
# Absolute request offset represented by token_ids[0].
token_ids_start: int = 0
num_prompt_tokens: int | None = None
# Total prefill tokens, including generated tokens replayed after preemption.
prefill_end_tokens: int | None = None
# Identifies this store job for the engine's lifetime. A request id cannot
# serve that purpose: it is reused once a preempted request resumes, so it
# would release the wrong job's blocks.
store_job_id: int | None = None
# Blocks stored under an explicit prefix hash rather than by position:
# core-provided mamba "align" states (a block-aligned entry is a committed
# snapshot, a non-aligned one the sub-block CoW prompt tail) and the
# scheduler-resolved non-Mamba blocks of the prompt's partial tail. The
# store-job reference keeps each block alive until every worker rank
# finishes the job.
boundary_puts: list[BoundaryPut] | None = None
# Total computed prefix length at the end of this step.
completed_token_len: int | None = None
# Set on the save that first covers the whole prompt, so the scheduler
# publishes the prompt's partial tail once rather than on every save.
publish_partial_tail: bool = False
@staticmethod
def from_request_tracker(
tracker: RequestTracker,
block_size: int,
load_spec: LoadSpec | None = None,
skip_save: bool | None = False,
block_hashes: list[BlockHash] | None = None,
save_partial_tail: bool = False,
num_prompt_tokens: int | None = None,
) -> "ReqMeta | None":
"""Create ReqMeta from a RequestTracker."""
if save_partial_tail:
assert num_prompt_tokens is not None
if block_hashes is None:
block_hashes = []
input_token_len = tracker.token_len
token_ids_start = tracker.num_saved_tokens
chunk_boundary = cdiv(token_ids_start + 1, block_size) * block_size
num_tokens_to_save = input_token_len // block_size * block_size
publish_tail = (
save_partial_tail
and not tracker.partial_tail_sent
and num_prompt_tokens is not None
and 0 < num_prompt_tokens <= input_token_len
)
skip_save = skip_save or (
num_tokens_to_save < chunk_boundary and not publish_tail
)
# A ReqMeta must never carry both a save AND a load.
# The save would also be wasted work — the bytes are being looked up
# in the store right now. Later cached_reqs steps save new tokens
# normally.
if load_spec is not None and load_spec.can_load:
skip_save = True
if skip_save and load_spec is None:
return None
if not skip_save:
tracker.num_saved_tokens = num_tokens_to_save
tracker.partial_tail_sent |= publish_tail
token_ids = None
if tracker.token_ids and not skip_save:
# Scheduler tracking continues while this job is handled by an
# asynchronous worker, so metadata must own a stable snapshot.
token_ids = tracker.token_ids[token_ids_start:num_tokens_to_save]
if load_spec is not None and load_spec.can_load:
logger.debug(
"Scheduled to load %d tokens for request %s",
load_spec.kvpool_cached_tokens,
tracker.req_id,
)
else:
load_spec = None
logger.debug(
"request:%s, meta save spec:%s, meta load spec:%s",
tracker.req_id,
not skip_save,
load_spec,
)
return ReqMeta(
req_id=tracker.req_id,
token_len_chunk=num_tokens_to_save,
block_ids=tracker.allocated_block_ids,
can_save=not skip_save,
load_spec=load_spec,
block_hashes=block_hashes,
token_ids=token_ids,
token_ids_start=token_ids_start,
num_prompt_tokens=num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
completed_token_len=input_token_len,
publish_partial_tail=publish_tail and not skip_save,
)
@dataclass
class MooncakeStoreWorkerMetadata(KVConnectorWorkerMetadata):
"""Maps ``ReqMeta.store_job_id`` to the number of ranks done with that job."""
completed_saves: dict[int, int] = field(default_factory=dict)
def aggregate(
self, other: "KVConnectorWorkerMetadata"
) -> "MooncakeStoreWorkerMetadata":
assert isinstance(other, MooncakeStoreWorkerMetadata)
for store_job_id, count in other.completed_saves.items():
self.completed_saves[store_job_id] = (
self.completed_saves.get(store_job_id, 0) + count
)
return self
class MooncakeStoreConnectorMetadata(KVConnectorMetadata):
"""Metadata passed from scheduler to worker."""
def __init__(
self,
unfinished_request_ids: set[str],
preempted_req_ids: set[str],
):
self.requests: list[ReqMeta] = []
self.unfinished_request_ids = unfinished_request_ids
self.preempted_req_ids = preempted_req_ids
def add_request(self, req_meta: ReqMeta) -> None:
self.requests.append(req_meta)
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-operation telemetry for MooncakeStoreConnector.
Records one row per Mooncake RPC (``save_exists``, ``save_put``, ``load_get``,
``lookup_exists``) with duration, key/byte counts, status, and failed-key
count. Exposed to the logger via ``KVConnectorLogging`` and to Prometheus
via ``MooncakeStorePromMetrics``.
"""
from dataclasses import dataclass
from statistics import fmean
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
def _nearest_rank_percentile(values: list[float], percentile: float) -> float:
if not values:
return 0.0
sorted_values = sorted(values)
rank = max(
0, min(len(sorted_values) - 1, int(percentile * len(sorted_values) - 1e-12))
)
return sorted_values[rank]
@dataclass
class MooncakeStoreConnectorStats(KVConnectorStats):
"""Serializable Mooncake store communication telemetry."""
def __post_init__(self):
if not self.data:
self.reset()
def reset(self):
self.data: dict[str, list[dict[str, int | float | str]]] = {}
def is_empty(self) -> bool:
return not self.data
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if other.is_empty():
return self
for operation, records in other.data.items():
self.data.setdefault(operation, []).extend(records)
return self
def reduce(self) -> dict[str, int | float]:
reduced: dict[str, int | float] = {}
for operation, records in sorted(self.data.items()):
if not records:
continue
durations = [float(record["duration_seconds"]) for record in records]
reduced[f"{operation}_count"] = len(records)
reduced[f"{operation}_avg_ms"] = round(fmean(durations) * 1e3, 3)
reduced[f"{operation}_p90_ms"] = round(
_nearest_rank_percentile(durations, 0.9) * 1e3, 3
)
reduced[f"{operation}_total_keys"] = sum(
int(record["num_keys"]) for record in records
)
reduced[f"{operation}_total_bytes"] = sum(
int(record["num_bytes"]) for record in records
)
reduced[f"{operation}_failed_keys"] = sum(
int(record["num_failed_keys"]) for record in records
)
reduced[f"{operation}_error_count"] = sum(
1 for record in records if record["status"] == "error"
)
return reduced
def record_operation(
self,
operation: str,
duration_seconds: float,
num_keys: int,
*,
num_bytes: int = 0,
status: str = "ok",
num_failed_keys: int = 0,
) -> None:
self.data.setdefault(operation, []).append(
{
"duration_seconds": duration_seconds,
"num_keys": num_keys,
"num_bytes": num_bytes,
"status": status,
"num_failed_keys": num_failed_keys,
}
)
class MooncakeStorePromMetrics(KVConnectorPromMetrics):
"""Prometheus metrics for Mooncake store communication."""
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
metric_labelnames = labelnames + ["operation", "status"]
self._metric_cache: dict[tuple[int, str, str], dict[str, PromMetric]] = {}
self._histogram_operation_time = self._histogram_cls(
name="vllm:mooncake_store_operation_time_seconds",
documentation="Histogram of Mooncake store communication time.",
buckets=[
1e-3,
5e-3,
1e-2,
5e-2,
1e-1,
2e-1,
3e-1,
4e-1,
5e-1,
7.5e-1,
1.0,
1.5,
2.0,
3.0,
4.0,
],
labelnames=metric_labelnames,
)
self._counter_operation_calls = self._counter_cls(
name="vllm:mooncake_store_operation_total",
documentation="Number of Mooncake store communication operations.",
labelnames=metric_labelnames,
)
self._counter_operation_keys = self._counter_cls(
name="vllm:mooncake_store_operation_keys_total",
documentation="Number of Mooncake store keys touched by operations.",
labelnames=metric_labelnames,
)
self._counter_operation_bytes = self._counter_cls(
name="vllm:mooncake_store_operation_bytes_total",
documentation="Number of bytes transferred by Mooncake store operations.",
labelnames=metric_labelnames,
)
self._counter_failed_keys = self._counter_cls(
name="vllm:mooncake_store_operation_failed_keys_total",
documentation="Number of Mooncake store keys that failed in operations.",
labelnames=metric_labelnames,
)
def _get_metrics(
self,
engine_idx: int,
operation: str,
status: str,
) -> dict[str, PromMetric]:
cache_key = (engine_idx, operation, status)
if cache_key not in self._metric_cache:
label_values = self.per_engine_labelvalues[engine_idx] + [operation, status]
self._metric_cache[cache_key] = {
"time": self._histogram_operation_time.labels(*label_values),
"calls": self._counter_operation_calls.labels(*label_values),
"keys": self._counter_operation_keys.labels(*label_values),
"bytes": self._counter_operation_bytes.labels(*label_values),
"failed_keys": self._counter_failed_keys.labels(*label_values),
}
return self._metric_cache[cache_key]
def observe(self, transfer_stats_data: dict[str, Any] | None, engine_idx: int = 0):
if not transfer_stats_data:
return
for operation, records in transfer_stats_data.items():
assert isinstance(records, list)
for record in records:
assert isinstance(record, dict)
status = str(record["status"])
metrics = self._get_metrics(engine_idx, operation, status)
metrics["time"].observe(float(record["duration_seconds"]))
metrics["calls"].inc()
metrics["keys"].inc(int(record["num_keys"]))
metrics["bytes"].inc(int(record["num_bytes"]))
metrics["failed_keys"].inc(int(record["num_failed_keys"]))
@@ -1,81 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Wire-format constants for the LookupKey ZMQ admin channel.
This is the single source of truth shared by ``LookupKeyClient`` and
``LookupKeyServer`` on the scheduler<->worker rank-0 admin channel.
Wire format (REQ/REP over IPC):
Request: [msg_type: bytes] [payload_frames...]
msg_type == LOOKUP_MSG:
frame 1: num_tokens (u32 big-endian, 4 bytes); the worker derives
the aligned lookup length
frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each
fixed-size block hash (0 when there are no hashes)
frame 3: raw block hashes concatenated back-to-back (each hash_len
bytes); the server splits on hash_len
Response: hit_length (u32 big-endian, first 4 bytes), followed by zero
or more 8-byte tail-key boundaries. Each entry is group_id
(u32), then boundary_tokens (u32).
msg_type == RESET_MSG:
(no payload frames)
Response: [RESP_OK] or [RESP_ERR]
The first frame of every request is a named bytes tag (not a numeric
sentinel that aliases the data field) so the protocol stays
self-describing and extensible: adding new admin commands requires
only a new tag and a new dispatch branch.
Mirrors the named-tag convention used by the NIXL connector (see
``vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py``).
"""
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
MooncakeLookupResult,
TailKeyBoundary,
)
# Request message-type tags. Frame 0 of every request.
LOOKUP_MSG: bytes = b"lookup"
RESET_MSG: bytes = b"reset"
# Single-byte response status codes for admin commands.
RESP_OK: bytes = b"\x01"
RESP_ERR: bytes = b"\x00"
# group_id (u32), boundary_tokens (u32).
TAIL_KEY_BOUNDARY_ENTRY_SIZE: int = 8
def encode_lookup_response(result: MooncakeLookupResult) -> bytes:
hit_length = result.hit_length.to_bytes(4, "big")
if not result.tail_key_boundaries:
return hit_length
payload = bytearray(hit_length)
for boundary in result.tail_key_boundaries:
payload.extend(boundary.group_id.to_bytes(4, "big"))
payload.extend(boundary.num_tokens.to_bytes(4, "big"))
return bytes(payload)
def decode_lookup_response(payload: bytes) -> MooncakeLookupResult:
if len(payload) < 4 or (len(payload) - 4) % TAIL_KEY_BOUNDARY_ENTRY_SIZE:
raise ValueError("Invalid Mooncake lookup response")
hit_length = int.from_bytes(payload[:4], "big")
if len(payload) == 4:
return MooncakeLookupResult(hit_length)
boundaries = []
for offset in range(4, len(payload), TAIL_KEY_BOUNDARY_ENTRY_SIZE):
boundaries.append(
TailKeyBoundary(
group_id=int.from_bytes(payload[offset : offset + 4], "big"),
num_tokens=int.from_bytes(payload[offset + 4 : offset + 8], "big"),
)
)
return MooncakeLookupResult(hit_length, tuple(boundaries))
@@ -1,788 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from vllm-project/vllm-ascend
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""Scheduler-side logic for MooncakeStoreConnector."""
from collections.abc import Sequence
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import (
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
BoundaryPut,
LoadSpec,
MooncakeStoreConnectorMetadata,
MooncakeStoreWorkerMetadata,
ReqMeta,
RequestTracker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
LookupKeyClient,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import (
resolve_kv_cache_block_sizes,
)
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
MambaSpec,
get_mamba_prefill_checkpoint_position,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
"""Tokens this prefill will compute KV for.
Under the v2 model runner, resumed-from-preemption requests appear in
``scheduled_new_reqs`` with ``prefill_token_ids`` set to the request's full
token list (prompt + previously-generated). For all other cases this falls
back to the original prompt.
"""
if request.prefill_token_ids is not None:
return request.prefill_token_ids
assert request.prompt_token_ids is not None
return request.prompt_token_ids
def _partial_tail_non_mamba_puts(
coord: MooncakeStoreCoordinator,
req_meta: ReqMeta,
block_sizes: Sequence[int],
) -> list[BoundaryPut]:
"""Locate the non-Mamba blocks a partial-tail save publishes.
A later request resumes at the prompt's Mamba checkpoint (``boundary``) only
if both pieces of KV are stored:
- Mamba groups: the state at ``boundary``, handed off by the core as a
boundary put of its own, so Mamba groups are not listed here.
- Other groups: the KV from the last LCM-aligned normal save up to
``boundary`` plus the group's EAGLE proof margin (``proof_end``). Each
block is keyed by the hash at its end, clipped to ``proof_end``.
Groups whose proof is not computed yet are omitted. For example, with LCM
16, blocks of 4 tokens, and a checkpoint at 44, the full-attention blocks
are 8-10, keyed at 36, 40 and 44.
A Mamba state handed off at any other position, i.e. a shared-prefix
junction (``--enable-mamba-shared-prefix-checkpoint``), adds no puts here.
``block_sizes`` are the per-group block sizes, indexed like
``req_meta.block_ids``.
"""
mamba_tails = [
put.num_tokens
for put in req_meta.boundary_puts or []
if put.group_id in coord.mamba_group_ids
and put.num_tokens % block_sizes[put.group_id]
]
# The tail is due on the save that first covers the prompt, or when Mamba
# hands off its checkpoint state.
if not mamba_tails and not req_meta.publish_partial_tail:
return []
prompt_tokens = req_meta.num_prompt_tokens
completed = req_meta.completed_token_len
assert prompt_tokens is not None and completed is not None
hash_block_size = coord.hash_block_size
# Core drops the Mamba checkpoint whenever EAGLE block drop is on, whichever
# groups carry the eagle flag.
boundary = get_mamba_prefill_checkpoint_position(
prompt_tokens, hash_block_size, coord.use_eagle
)
# Any other sub-block Mamba hand-off is a shared-prefix junction. Its
# attention KV, EAGLE proof included, ends on an attention block end no later
# than this checkpoint's proof, so the normal and prompt-tail saves store it.
if not req_meta.publish_partial_tail and boundary not in mamba_tails:
return []
num_hashes = len(req_meta.block_hashes)
if boundary == 0 or boundary // hash_block_size > num_hashes:
return []
start = boundary // coord.lcm_block_size * coord.lcm_block_size
puts: list[BoundaryPut] = []
for group_id, block_size in enumerate(block_sizes):
if group_id in coord.mamba_group_ids:
continue
proof_end = boundary + coord.eagle_proof_margin_by_group.get(group_id, 0)
if proof_end > completed or proof_end // hash_block_size > num_hashes:
continue
group_blocks = req_meta.block_ids[group_id]
for block_idx in range(start // block_size, cdiv(proof_end, block_size)):
if block_idx >= len(group_blocks):
break
block_id = group_blocks[block_idx]
if block_id == NULL_BLOCK_ID:
continue
num_tokens = min((block_idx + 1) * block_size, proof_end)
puts.append(BoundaryPut(group_id, block_id, num_tokens))
return puts
class MooncakeStoreScheduler:
"""Scheduler-side component for MooncakeStoreConnector."""
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
):
assert vllm_config.kv_transfer_config is not None
self.kv_role = vllm_config.kv_transfer_config.kv_role
kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
self.load_async = kvc_extra_config.get("load_async", True)
self.lookup_async = kvc_extra_config.get("lookup_async", False)
# Skips lookup CPU cost on instances that never load KV from the store.
self.enable_lookup = kvc_extra_config.get("enable_lookup", True)
self.save_decode_cache = kvc_extra_config.get("save_decode_cache", False)
kv_event_config = vllm_config.kv_events_config
self.enable_kv_events = bool(
kv_event_config and kv_event_config.enable_kv_cache_events
)
self.client = LookupKeyClient(vllm_config)
self.kv_cache_config = kv_cache_config
self._store_group_ids = kv_cache_config.prefix_cacheable_group_ids
# Map scheduler group IDs to store group indices. Groups outside the
# store projection, such as the QSA ring, map to None and are skipped.
self._store_group_id_by_kv_cache_group_id = {
group_id: store_group_id
for store_group_id, group_id in enumerate(self._store_group_ids)
}
store_groups = kv_cache_config.prefix_cacheable_groups
# Align with the engine's own scheduler_block_size and hash_block_size.
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
mamba_groups = {
group_id: group.kv_cache_spec
for group_id, group in enumerate(store_groups)
if isinstance(group.kv_cache_spec, MambaSpec)
}
self._boundary_state_group_ids = frozenset(mamba_groups)
# Mirrors MooncakeStoreConnector._capacity_only.
self._capacity_only = (
self.kv_role == "kv_consumer"
and not self.enable_lookup
and not self.save_decode_cache
)
self._store_coord: MooncakeStoreCoordinator | None = None
self.enable_partial_hash_hits = False
if not self._capacity_only:
assert all(
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
)
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
self._gpu_block_pool: BlockPool | None = None
self._num_workers = vllm_config.parallel_config.world_size
self._next_store_job_id = 0
# store_job_id -> (referenced block ids, ranks yet to report completion)
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
# Per-request state
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
self._finished_partial_tail_metas: dict[str, ReqMeta] = {}
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
self._gpu_block_pool = gpu_block_pool
def get_num_new_matched_tokens(
self,
request: Request,
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""Check for external KV cache hit.
Returns ``(None, False)`` when an async lookup is still in flight,
signaling the scheduler to retry this request on a later step.
"""
if not self.enable_lookup:
return 0, False
# Fine-grained hits may land on a hash boundary inside a block; without
# partial hits, prefixes shorter than one physical block are skipped.
align = (
self._hash_block_size if self.enable_partial_hash_hits else self._block_size
)
if request.num_tokens < align:
return 0, False
lookup_result = self.client.lookup(
request.request_id,
request.num_tokens,
request.block_hashes,
non_block=self.lookup_async,
)
if lookup_result is None:
# Lookup not ready yet; scheduler will retry on a later step.
return None, False
num_external_hit_tokens = lookup_result.hit_length
if num_external_hit_tokens < num_computed_tokens:
need_to_allocate = 0
else:
need_to_allocate = num_external_hit_tokens - num_computed_tokens
logger.debug(
"Reqid: %s, Total tokens %d, kvpool hit tokens: %d, need to load: %d",
request.request_id,
request.num_tokens,
num_external_hit_tokens,
need_to_allocate,
)
if need_to_allocate <= 0:
return 0, False
self.load_specs[request.request_id] = LoadSpec(
vllm_cached_tokens=num_computed_tokens,
kvpool_cached_tokens=num_external_hit_tokens,
can_load=False,
tail_key_boundaries=lookup_result.tail_key_boundaries,
)
return need_to_allocate, self.load_async
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
):
"""Update state after block allocation."""
local_block_ids: tuple[list[int], ...] = ()
if num_external_tokens > 0:
local_block_ids = blocks.get_block_ids(
group_ids=self._store_group_ids,
)
self._unfinished_requests[request.request_id] = (request, local_block_ids)
self._unfinished_request_ids.add(request.request_id)
if request.request_id not in self.load_specs:
return
if num_external_tokens == 0:
self.load_specs[request.request_id].can_load = False
return
assert (
num_external_tokens > 0
and num_external_tokens
== self.load_specs[request.request_id].kvpool_cached_tokens
- self.load_specs[request.request_id].vllm_cached_tokens
), (
f"Mismatch in number of tokens: {num_external_tokens} vs "
f"{self.load_specs[request.request_id].kvpool_cached_tokens} - "
f"{self.load_specs[request.request_id].vllm_cached_tokens}"
f" for request {request.request_id}"
)
self.load_specs[request.request_id].can_load = True
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""Build connector metadata for this scheduler step."""
is_consumer = self.kv_role == "kv_consumer"
can_process_cached = not is_consumer or self.save_decode_cache
for finished_req_id in scheduler_output.finished_req_ids:
self.client.discard(finished_req_id)
self.load_specs.pop(finished_req_id, None)
self._request_trackers.pop(finished_req_id, None)
self._unfinished_requests.pop(finished_req_id, None)
self._unfinished_request_ids.discard(finished_req_id)
preempted_ids = scheduler_output.preempted_req_ids or set()
for req_id in preempted_ids:
self.load_specs.pop(req_id, None)
if request_tracker := self._request_trackers.get(req_id):
request_tracker.reset()
self._unfinished_requests.pop(req_id, None)
meta = MooncakeStoreConnectorMetadata(
self._unfinished_request_ids,
preempted_ids,
)
# Handle new requests
for request in scheduler_output.scheduled_new_reqs:
load_spec = self.load_specs.pop(request.req_id, None)
num_tokens_to_compute = (
request.num_computed_tokens
+ scheduler_output.num_scheduled_tokens[request.req_id]
)
assert request.req_id in self._unfinished_requests
request_tuple = self._unfinished_requests.get(request.req_id)
request_real = request_tuple[0] # type: ignore[index]
unfolded_block_ids = tuple(
request.block_ids[group_id].copy() for group_id in self._store_group_ids
)
prefill_tokens = _new_req_prefill_tokens(request)
request_tracker = RequestTracker(
req_id=request.req_id,
token_len=num_tokens_to_compute,
allocated_block_ids=unfolded_block_ids,
num_saved_tokens=0,
token_ids=(
prefill_tokens[:num_tokens_to_compute]
if self.enable_kv_events
else None
),
prefill_end_tokens=len(prefill_tokens),
)
self._request_trackers[request.req_id] = request_tracker
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
# A consumer may write decode KV without becoming a prefill
# producer. Loads are still carried by the same metadata.
skip_save=is_consumer,
block_hashes=request_real.block_hashes,
num_prompt_tokens=request_real.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
if req_meta is not None:
meta.add_request(req_meta)
# Handle cached (running, or MRV1 resumed-from-preemption) requests
cached_reqs = scheduler_output.scheduled_cached_reqs
if can_process_cached:
for i, req_id in enumerate(cached_reqs.req_ids):
new_block_ids = cached_reqs.new_block_ids[i]
if new_block_ids:
new_block_ids = tuple(
new_block_ids[group_id] for group_id in self._store_group_ids
)
req_meta = None
if req_id in cached_reqs.resumed_req_ids:
# Resumed after preemption
if not new_block_ids:
continue
new_block_ids = tuple(b.copy() for b in new_block_ids)
load_spec = self.load_specs.pop(req_id, None)
request_tuple = self._unfinished_requests.get(req_id)
request_real = request_tuple[0] # type: ignore[index]
num_tokens_to_compute = (
request_real.num_computed_tokens
+ scheduler_output.num_scheduled_tokens[req_id]
)
# On resume, the request re-prefills prompt + previously
# generated tokens (all_token_ids).
prefill_tokens = list(request_real.all_token_ids)
request_tracker = RequestTracker(
req_id=req_id,
token_len=num_tokens_to_compute,
allocated_block_ids=new_block_ids,
num_saved_tokens=0,
token_ids=(
prefill_tokens[:num_tokens_to_compute]
if self.enable_kv_events
else None
),
prefill_end_tokens=len(prefill_tokens),
)
self._request_trackers[req_id] = request_tracker
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
skip_save=is_consumer,
block_hashes=request_real.block_hashes,
num_prompt_tokens=request_real.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
else:
# Decode/chunked request
request_tracker = self._request_trackers[req_id]
num_computed_token = cached_reqs.num_computed_tokens[i]
# Use the tracker's snapshot of the prefill range so resumed
# requests keep saving past the original prompt boundary.
prefill_end = request_tracker.prefill_end_tokens
is_decode = num_computed_token >= prefill_end
if is_decode and not self.save_decode_cache:
continue
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
req_tuple = self._unfinished_requests.get(req_id)
if req_tuple:
unfinished_req = req_tuple[0]
num_current_tokens = request_tracker.token_len
new_token_ids = unfinished_req.all_token_ids[
num_current_tokens : num_current_tokens + num_new_tokens
]
request_tracker.token_len += len(new_token_ids)
if request_tracker.token_ids is not None:
request_tracker.token_ids.extend(new_token_ids)
else:
raise ValueError(
f"Request {req_id} is not in _unfinished_requests"
)
# A block is usually allocated before the step that fills
# it, so reaching a save boundary does not imply that this
# step has new block ids.
if new_block_ids:
request_tracker.update(new_block_ids)
if is_consumer and not is_decode:
continue
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=None,
skip_save=False,
block_hashes=unfinished_req.block_hashes,
num_prompt_tokens=unfinished_req.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
if req_meta is not None:
meta.add_request(req_meta)
# Handle requests with pending load specs not yet scheduled
request_ids = [req.req_id for req in scheduler_output.scheduled_new_reqs]
for request_id, (
unfinished_req,
block_ids,
) in self._unfinished_requests.items():
if request_id not in request_ids and request_id not in cached_reqs.req_ids:
load_spec = self.load_specs.pop(request_id, None)
# A load spec may have been proposed by this connector's
# lookup but rejected by MultiConnector in favor of another
# connector. Only the chosen connector may issue the pending
# load; the normal store path gets its blocks later from
# SchedulerOutput once the request is actually scheduled.
if load_spec is None or not load_spec.can_load:
continue
num_tokens_to_compute = load_spec.kvpool_cached_tokens
request_tracker = RequestTracker(
req_id=request_id,
token_len=num_tokens_to_compute,
allocated_block_ids=block_ids,
num_saved_tokens=0,
)
self._request_trackers[request_id] = request_tracker
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
skip_save=None,
block_hashes=unfinished_req.block_hashes,
num_prompt_tokens=unfinished_req.num_prompt_tokens,
)
if req_meta is not None:
meta.add_request(req_meta)
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
if (
block_state is not None
and block_state.boundary_state_offloads
and not is_consumer
):
self._handle_boundary_state_offloads(
block_state.boundary_state_offloads, meta
)
self._apply_current_save_block_ids(meta, scheduler_output)
# Finish-time handoffs arrive after the producing step's metadata was
# built. Their exact blocks were pinned when they were registered, so
# they remain valid after request cleanup and need no current snapshot.
for req_meta in self._finished_partial_tail_metas.values():
meta.add_request(req_meta)
self._finished_partial_tail_metas.clear()
if not self._capacity_only:
self._reference_save_blocks(meta)
return meta
def _apply_current_save_block_ids(
self,
meta: MooncakeStoreConnectorMetadata,
scheduler_output: SchedulerOutput,
) -> None:
"""Replace append-only mirrors with the core's current block tables."""
save_metas = [req_meta for req_meta in meta.requests if req_meta.can_save]
if not save_metas:
return
block_state = scheduler_output.kv_connector_block_state
assert block_state is not None, (
"Current block tables are required for Mooncake store jobs"
)
for req_meta in save_metas:
block_ids = block_state.get_block_ids(req_meta.req_id)
assert block_ids is not None, (
f"Missing current block table for store request {req_meta.req_id}"
)
req_meta.block_ids = tuple(
block_ids[group_id] for group_id in self._store_group_ids
)
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
"""Take a GPU block reference for every store job this step emits.
The worker DMAs out of these blocks after the step that scheduled them,
so a reference keeps them out of the free queue even once the request
itself is freed, until every rank reports the job done.
"""
pool = self._gpu_block_pool
coord = self._store_coord
assert coord is not None
block_sizes = [g.kv_cache_spec.block_size for g in coord.kv_cache_groups]
for req_meta in meta.requests:
if not req_meta.can_save:
continue
assert pool is not None, (
"GPU block pool must be bound before any store job is emitted"
)
if req_meta.store_job_id is not None:
assert req_meta.store_job_id in self._pinned_saves
continue
if tail := _partial_tail_non_mamba_puts(coord, req_meta, block_sizes):
req_meta.boundary_puts = [*tail, *(req_meta.boundary_puts or [])]
req_meta.store_job_id = store_job_id = self._next_store_job_id
self._next_store_job_id += 1
block_ids = [put.block_id for put in req_meta.boundary_puts or []]
assert NULL_BLOCK_ID not in block_ids, (
"A null block cannot back a boundary put"
)
if req_meta.token_len_chunk:
# Normal prefix save: pin all non-Mamba sources for retries.
# Every allocated block is referenced, not just the ones covering
# this job's token range: a rank resumes from its own last
# successful offset, which lags the scheduler's whenever a save was
# skipped or failed, so it may read anywhere below the range.
block_ids.extend(
block_id
for group_id, group in enumerate(req_meta.block_ids)
if group_id not in self._boundary_state_group_ids
for block_id in group
if block_id != NULL_BLOCK_ID
)
# An aligned boundary block may also be present in the request's
# block table. Take and release exactly one reference per block.
block_ids = list(dict.fromkeys(block_ids))
assert NULL_BLOCK_ID not in block_ids
if not block_ids:
continue
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
pool.touch([pool.blocks[block_id] for block_id in block_ids])
def register_finished_partial_tail(
self,
request: Request,
block_ids: tuple[list[int], ...],
partial_tail_offloads: list[tuple[int, int, int]],
) -> bool:
"""Queue and pin a finish-time tail for the next connector step."""
if self.kv_role == "kv_consumer" or not partial_tail_offloads:
return False
tracker = self._request_trackers.get(request.request_id)
if tracker is None or not any(block_ids):
return False
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
if len(boundaries) != 1:
raise ValueError(
"Partial-tail offloads for one request must share a boundary"
)
boundary_tokens = next(iter(boundaries))
if boundary_tokens > tracker.prefill_end_tokens:
return False
mamba_puts: list[BoundaryPut] = []
for group_id, block_id, boundary in partial_tail_offloads:
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
if store_group_id not in self._boundary_state_group_ids:
return False
if block_id == NULL_BLOCK_ID:
return False
mamba_puts.append(BoundaryPut(store_group_id, block_id, boundary))
req_meta = ReqMeta(
req_id=request.request_id,
token_len_chunk=0,
block_ids=tuple(
block_ids[group_id].copy() for group_id in self._store_group_ids
),
block_hashes=list(request.block_hashes),
can_save=True,
num_prompt_tokens=request.num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
boundary_puts=mamba_puts,
completed_token_len=request.num_computed_tokens,
)
coord = self._store_coord
assert coord is not None
req_meta.boundary_puts = [
*_partial_tail_non_mamba_puts(
coord,
req_meta,
[g.kv_cache_spec.block_size for g in coord.kv_cache_groups],
),
*mamba_puts,
]
pinned_block_ids = list(
dict.fromkeys(put.block_id for put in req_meta.boundary_puts)
)
pool = self._gpu_block_pool
assert pool is not None, (
"GPU block pool must be bound before a finish-time handoff"
)
assert request.request_id not in self._finished_partial_tail_metas
store_job_id = self._next_store_job_id
self._next_store_job_id += 1
self._pinned_saves[store_job_id] = (pinned_block_ids, self._num_workers)
pool.touch([pool.blocks[block_id] for block_id in pinned_block_ids])
req_meta.store_job_id = store_job_id
self._finished_partial_tail_metas[request.request_id] = req_meta
tracker.has_pending_offload = True
# The store job owns exact block refs, so request cleanup need not wait.
return False
def _handle_boundary_state_offloads(
self,
offloads: dict[str, list[tuple[int, int, int]]],
meta: MooncakeStoreConnectorMetadata,
) -> None:
"""Attach exact boundary-state blocks to store jobs for this step.
Flushed in the step they arrive: the CoW copy is enqueued before the
connector event records, so this step's event fences the exact block.
Entries ride the request's save meta when present, else an
offload-only ReqMeta (``token_len_chunk=0`` skips the normal save,
``can_save=True`` takes the normal enqueue and store-job pinning path).
"""
save_metas = {m.req_id: m for m in meta.requests if m.can_save}
for req_id, entries in offloads.items():
tracker = self._request_trackers.get(req_id)
req_tuple = self._unfinished_requests.get(req_id)
if tracker is None or req_tuple is None:
# Request finished/preempted within this step; its blocks are
# going away, so the offload is conservatively dropped.
logger.debug("Dropping boundary-state offload for request %s", req_id)
continue
accepted: list[BoundaryPut] = []
for group_id, block_id, boundary_tokens in entries:
# Every other group stops saving at the end of this prefill, so
# a mamba-only key past it can never complete a joint hybrid
# hit. `prefill_end_tokens` — not the original prompt length —
# is the boundary: a resumed request re-prefills and re-saves
# its previously generated tokens for every group.
if boundary_tokens > tracker.prefill_end_tokens:
continue
if block_id == NULL_BLOCK_ID:
continue
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
if store_group_id not in self._boundary_state_group_ids:
continue
accepted.append(BoundaryPut(store_group_id, block_id, boundary_tokens))
if not accepted:
continue
tracker.has_pending_offload = True
if (req_meta := save_metas.get(req_id)) is not None:
req_meta.boundary_puts = accepted
continue
meta.add_request(
ReqMeta(
req_id=req_id,
token_len_chunk=0,
block_ids=tracker.allocated_block_ids,
block_hashes=req_tuple[0].block_hashes,
can_save=True,
num_prompt_tokens=req_tuple[0].num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
boundary_puts=accepted,
completed_token_len=tracker.token_len,
)
)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Drop the block references of store jobs every rank has finished."""
meta = connector_output.kv_connector_worker_meta
if not isinstance(meta, MooncakeStoreWorkerMetadata):
return
pool = self._gpu_block_pool
assert pool is not None
for store_job_id, count in meta.completed_saves.items():
pinned = self._pinned_saves.get(store_job_id)
if pinned is None:
# The job referenced no blocks, so nothing was recorded for it.
continue
block_ids, remaining = pinned
remaining -= count
if remaining > 0:
self._pinned_saves[store_job_id] = (block_ids, remaining)
continue
assert remaining == 0, (
f"store job {store_job_id} reported by too many ranks"
)
del self._pinned_saves[store_job_id]
# Tail-first, as elsewhere, so the shared prefix is evicted last.
pool.free_blocks(pool.blocks[bid] for bid in reversed(block_ids))
def has_pending_push_work(self) -> bool:
"""Keep the engine stepping while any store job still holds block refs.
Completions only reach the scheduler as worker metadata on a step, so an
engine that quiesced with jobs in flight would leave those references
held indefinitely. Nothing else keeps it alive now that a finishing
request no longer defers its own free.
"""
return bool(self._pinned_saves)
def reset_store(self) -> bool:
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
Routes through the existing LookupKey ZMQ admin channel to worker
rank 0, which owns the ``MooncakeDistributedStore`` handle.
Ordering assumption: caller (typically
``Scheduler.reset_connector_cache``, invoked via
``reset_prefix_cache(reset_connector=True)``) MUST ensure no
in-flight Mooncake lookups or transfers. For RL workflows this is
satisfied at the step boundary after weight updates and rollout
drain. Violating this can allow stale KV to be served on the next
request, defeating the hard-reset guarantee.
Returns True on ACK from worker, False on NACK or RPC error.
"""
try:
ok = self.client.reset()
if ok:
logger.info("Mooncake store reset via remove_all succeeded.")
else:
logger.warning("Mooncake store reset returned NACK from worker.")
return ok
except Exception as e:
logger.error("Mooncake reset_store RPC failed: %s", e)
return False
File diff suppressed because it is too large Load Diff
@@ -1,41 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .connector import UMBPStoreConnector
from .data import (
BlockIdentityCodec,
BlockTransferPlan,
KVLayoutDescriptor,
KVLayoutPlanner,
KVRange,
KVRegion,
LoadSpec,
RankTopology,
RequestTracker,
TransferJobState,
TransferJobStatus,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
UMBPNamespace,
)
from .runtime import UMBPRuntimeConfig, UMBPRuntimeFactory
__all__ = [
"UMBPStoreConnector",
"BlockIdentityCodec",
"BlockTransferPlan",
"KVLayoutPlanner",
"KVLayoutDescriptor",
"KVRange",
"KVRegion",
"LoadSpec",
"RequestTracker",
"RankTopology",
"TransferJobState",
"TransferJobStatus",
"UMBPConnectorMetadata",
"UMBPConnectorWorkerMetadata",
"UMBPNamespace",
"UMBPRuntimeConfig",
"UMBPRuntimeFactory",
]
@@ -1,220 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM-facing shared UMBP connector."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
)
from vllm.forward_context import ForwardContext
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from .data import (
BlockIdentityCodec,
KVLayoutPlanner,
RankTopology,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
UMBPNamespace,
)
from .runtime import UMBPRuntimeConfig, UMBPRuntimeFactory
from .scheduler import UMBPStoreConnectorScheduler
from .worker import UMBPStoreConnectorWorker
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
class UMBPStoreConnector(KVConnectorBase_V1):
"""vLLM connector lifecycle shared by every UMBP runtime mode."""
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig,
) -> None:
super().__init__(vllm_config, role, kv_cache_config)
topology = RankTopology.from_vllm_config(vllm_config)
runtime_config = UMBPRuntimeConfig.from_vllm(
vllm_config
).resolve_for_rank_count(topology.rank_count)
if topology.pp_size > 1:
raise NotImplementedError(
f"{runtime_config.mode} UMBP does not support pipeline parallelism"
)
runtime = UMBPRuntimeFactory.build(runtime_config)
namespace = UMBPNamespace.from_vllm_config(vllm_config, kv_cache_config)
codec = BlockIdentityCodec(
namespace=namespace,
tp_rank=topology.tp_rank,
pp_rank=topology.pp_rank,
pcp_rank=topology.pcp_rank,
dcp_rank=topology.dcp_rank,
)
layout = KVLayoutPlanner.from_kv_cache_config(kv_cache_config)
layout_descriptor = layout.describe(topology)
self._runtime = runtime
self._store_submitted = False
self.connector_scheduler: UMBPStoreConnectorScheduler | None = None
self.connector_worker: UMBPStoreConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = UMBPStoreConnectorScheduler(
vllm_config,
kv_cache_config,
runtime.create_scheduler_handle(
namespace.value, topology, layout_descriptor
),
codec,
topology,
)
else:
self.connector_worker = UMBPStoreConnectorWorker(
runtime.create_worker_handle(
namespace.value, topology, layout_descriptor
),
layout,
codec=codec,
)
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_state_after_alloc(
request, blocks, num_external_tokens
)
def bind_gpu_block_pool(self, gpu_block_pool: Any) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def request_finished(
self, request: Request, block_ids: list[int]
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, (block_ids,))
def update_connector_output(self, connector_output: Any) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_pending_push_work()
def reset_cache(self) -> bool | None:
if self.connector_scheduler is None:
return None
return self.connector_scheduler.reset_store()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
super().bind_connector_metadata(connector_metadata)
self._store_submitted = False
def clear_connector_metadata(self) -> None:
# No-forward steps skip the model runner's wait_for_save hook.
if self.connector_worker is not None and self.has_connector_metadata():
self.wait_for_save()
super().clear_connector_metadata()
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata) -> None:
assert self.connector_worker is not None
assert isinstance(kv_connector_metadata, UMBPConnectorMetadata)
self.connector_worker.handle_preemptions(kv_connector_metadata)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
del kwargs
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, UMBPConnectorMetadata)
self.connector_worker.start_load_kv(forward_context, metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
assert self.connector_worker is not None
self.connector_worker.wait_for_layer_load(layer_name)
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
# Stores are submitted once per step, from wait_for_save.
return
def wait_for_save(self) -> None:
if self._store_submitted:
return
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, UMBPConnectorMetadata)
self.connector_worker.enqueue_stores(metadata)
self.connector_worker.wait_for_save()
self._store_submitted = True
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
assert self.connector_worker is not None
return self.connector_worker.get_finished(finished_req_ids)
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
finished_sending, finished_recving = self.connector_worker.get_finished(
finished_req_ids
)
return KVConnectorTransferResults(
finished_sending=set(finished_sending or ()),
finished_recving=set(finished_recving or ()),
)
def get_block_ids_with_load_errors(self) -> set[int]:
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
def build_connector_worker_meta(self) -> UMBPConnectorWorkerMetadata:
assert self.connector_worker is not None
return self.connector_worker.build_connector_worker_meta()
def shutdown(self) -> None:
if self.connector_scheduler is not None:
self.connector_scheduler.close()
if self.connector_worker is not None:
self.connector_worker.close()
@@ -1,620 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared data types for the UMBP KV connector.
The types in this module deliberately do not depend on a UMBP client. They
describe vLLM's logical KV objects and the physical ranges a runtime adapter
will transfer.
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Sequence
from dataclasses import dataclass, field, replace
from enum import Enum
from typing import Any, NamedTuple
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.v1.kv_cache_interface import KVCacheConfig
@dataclass(frozen=True)
class RankTopology:
"""Model-parallel ranks that participate in one KV object."""
tp_rank: int = 0
tp_size: int = 1
pp_rank: int = 0
pp_size: int = 1
pcp_rank: int = 0
pcp_size: int = 1
dcp_rank: int = 0
dcp_size: int = 1
def __post_init__(self) -> None:
for name in ("tp_size", "pp_size", "pcp_size", "dcp_size"):
if getattr(self, name) <= 0:
raise ValueError(f"{name} must be positive")
for rank, size, name in (
(self.tp_rank, self.tp_size, "tp_rank"),
(self.pp_rank, self.pp_size, "pp_rank"),
(self.pcp_rank, self.pcp_size, "pcp_rank"),
(self.dcp_rank, self.dcp_size, "dcp_rank"),
):
if rank < 0 or rank >= size:
raise ValueError(f"{name}={rank} is outside [0, {size})")
@classmethod
def from_vllm_config(cls, vllm_config: Any) -> RankTopology:
parallel = vllm_config.parallel_config
tp_size = getattr(parallel, "tensor_parallel_size", 1)
pp_size = getattr(parallel, "pipeline_parallel_size", 1)
pcp_size = getattr(parallel, "prefill_context_parallel_size", 1)
# An engine's workers are laid out as [PP, PCP, TP], TP innermost
# (initialize_model_parallel).
rank = getattr(parallel, "rank", 0)
tp_rank = rank % tp_size
dcp_size = getattr(parallel, "decode_context_parallel_size", 1)
return cls(
tp_rank=tp_rank,
tp_size=tp_size,
pp_rank=rank // (tp_size * pcp_size) % pp_size,
pp_size=pp_size,
pcp_rank=rank // tp_size % pcp_size,
pcp_size=pcp_size,
# DCP partitions the TP workers; it does not create a second
# Cartesian worker dimension. ParallelConfig is constructed before
# process groups, so derive this stable identity from TP rank.
dcp_rank=tp_rank % dcp_size,
dcp_size=dcp_size,
)
@property
def local_namespace(self) -> tuple[int, int, int, int]:
return (self.tp_rank, self.pcp_rank, self.dcp_rank, self.pp_rank)
@property
def rank_count(self) -> int:
return len(self.all_namespaces())
def all_namespaces(self) -> tuple[tuple[int, int, int, int], ...]:
if self.dcp_size > 1:
if self.tp_size % self.dcp_size:
raise ValueError("dcp_size must divide tp_size")
return tuple(
(tp, pcp, tp % self.dcp_size, pp)
for pp in range(self.pp_size)
for pcp in range(self.pcp_size)
for tp in range(self.tp_size)
)
return tuple(
(tp, pcp, dcp, pp)
for pp in range(self.pp_size)
for dcp in range(self.dcp_size)
for pcp in range(self.pcp_size)
for tp in range(self.tp_size)
)
@dataclass(frozen=True)
class UMBPNamespace:
"""Compatibility namespace for objects stored outside the engine."""
value: str
@classmethod
def from_vllm_config(
cls,
vllm_config: Any,
kv_cache_config: KVCacheConfig,
) -> UMBPNamespace:
extra = vllm_config.kv_transfer_config.kv_connector_extra_config
configured = extra.get("key_namespace", "auto")
if configured != "auto":
if not isinstance(configured, str) or not configured:
raise ValueError("key_namespace must be a non-empty string or 'auto'")
return cls(configured)
model_config = vllm_config.model_config
parallel_config = vllm_config.parallel_config
# Draft-model KV (e.g. DSpark/EAGLE layers) is stored alongside the
# target's, so a different drafter must not reuse it.
speculative_config = getattr(vllm_config, "speculative_config", None)
draft_model_config = getattr(speculative_config, "draft_model_config", None)
speculative = (
None
if speculative_config is None
else {
"method": getattr(speculative_config, "method", None),
"model": getattr(draft_model_config, "model", None),
"revision": getattr(draft_model_config, "revision", None),
}
)
fields = {
"model": getattr(model_config, "model", ""),
"revision": getattr(model_config, "revision", None),
"block_size": vllm_config.cache_config.block_size,
"cache_layout": getattr(kv_cache_config, "kv_cache_layout", None),
"groups": [
{
"layers": group.layer_names,
"block_size": group.kv_cache_spec.block_size,
"kind": type(group.kv_cache_spec).__name__,
"dtype": str(getattr(group.kv_cache_spec, "dtype", None)),
}
for group in kv_cache_config.kv_cache_groups
],
"tp": parallel_config.tensor_parallel_size,
"pp": parallel_config.pipeline_parallel_size,
"pcp": getattr(parallel_config, "prefill_context_parallel_size", 1),
"dcp": getattr(parallel_config, "decode_context_parallel_size", 1),
"layout_version": 1,
}
if speculative is not None:
# Added only when set, so namespaces without a drafter are unchanged.
fields["speculative"] = speculative
payload = json.dumps(fields, sort_keys=True, separators=(",", ":")).encode()
return cls(hashlib.sha256(payload).hexdigest()[:32])
@dataclass(frozen=True)
class BlockIdentityCodec:
"""Build stable rank-local keys from vLLM block hashes."""
namespace: UMBPNamespace
tp_rank: int = 0
pp_rank: int = 0
pcp_rank: int = 0
dcp_rank: int = 0
def key(self, block_hash: bytes, group_id: int = 0) -> str:
return self.key_for_namespace(
block_hash,
group_id,
(self.tp_rank, self.pcp_rank, self.dcp_rank, self.pp_rank),
)
def key_for_namespace(
self,
block_hash: bytes,
group_id: int,
namespace: tuple[int, int, int, int],
) -> str:
if not isinstance(block_hash, bytes) or not block_hash:
raise ValueError("block_hash must be non-empty bytes")
if group_id < 0:
raise ValueError("group_id must be non-negative")
tp_rank, pcp_rank, dcp_rank, pp_rank = namespace
return (
f"umbp:vllm:v1:{self.namespace.value}:"
f"tp{tp_rank}:pcp{pcp_rank}:dcp{dcp_rank}:"
f"pp{pp_rank}:g{group_id}:"
f"{block_hash.hex()}"
)
def keys_for_topology(
self,
block_hash: bytes,
topology: RankTopology,
group_ids: Sequence[int],
) -> tuple[str, ...]:
return tuple(
self.key_for_namespace(block_hash, group_id, namespace)
for namespace in topology.all_namespaces()
for group_id in group_ids
)
@dataclass(frozen=True)
class KVRegion:
"""One deterministic region in a rank-local KV object."""
layer_name: str
group_id: int
block_stride: int
block_bytes: int
object_offset: int
block_size: int = 1
def __post_init__(self) -> None:
if self.block_stride <= 0 or self.block_bytes <= 0:
raise ValueError("KV region sizes must be positive")
if self.block_size <= 0:
raise ValueError("KV region block_size must be positive")
if self.block_bytes > self.block_stride:
raise ValueError(
f"KV region bytes ({self.block_bytes}) exceed its stride "
f"({self.block_stride}) for {self.layer_name!r}"
)
@dataclass(frozen=True)
class KVLayoutDescriptor:
"""Backend-neutral description of one rank-local KV object layout."""
regions: tuple[KVRegion, ...]
topology: RankTopology
layout_format: str = "rank_local"
def __post_init__(self) -> None:
if not self.layout_format:
raise ValueError("layout_format must not be empty")
names = [region.layer_name for region in self.regions]
if len(names) != len(set(names)):
raise ValueError("KV layout contains duplicate layer regions")
@property
def object_size(self) -> int:
return sum(region.block_bytes for region in self.regions)
class KVLayoutPlanner:
"""Create a deterministic all-layer object layout from KVCacheConfig."""
def __init__(self, regions: Sequence[KVRegion]) -> None:
self.regions = tuple(regions)
self._regions_by_group = {
group_id: tuple(r for r in self.regions if r.group_id == group_id)
for group_id in {r.group_id for r in self.regions}
}
self._base_addresses: dict[str, int] = {}
self._physical_strides: dict[str, int] = {}
@classmethod
def from_kv_cache_config(cls, config: KVCacheConfig) -> KVLayoutPlanner:
group_by_layer = {
layer: group_id
for group_id, group in enumerate(config.kv_cache_groups)
for layer in group.layer_names
if group.enable_kv_transfer
}
regions: list[KVRegion] = []
offset = 0
for tensor in config.kv_cache_tensors:
for layer_name in tensor.layers:
group_id = group_by_layer.get(layer_name)
if group_id is None:
continue
spec = config.kv_cache_groups[group_id].kv_cache_spec
block_bytes = spec.page_size_bytes
regions.append(
KVRegion(
layer_name=layer_name,
group_id=group_id,
block_stride=tensor.block_stride,
block_bytes=block_bytes,
object_offset=offset,
block_size=spec.block_size,
)
)
offset += block_bytes
return cls(regions)
@property
def object_size(self) -> int:
return sum(region.block_bytes for region in self.regions)
def describe(
self,
topology: RankTopology,
layout_format: str = "rank_local",
) -> KVLayoutDescriptor:
return KVLayoutDescriptor(
regions=self.regions,
topology=topology,
layout_format=layout_format,
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
"""Capture stable base addresses for the planned KV regions."""
required = {region.layer_name for region in self.regions}
missing = required - kv_caches.keys()
if missing:
raise ValueError(f"missing KV cache layers: {sorted(missing)}")
addresses: dict[str, int] = {}
for layer_name in required:
cache = kv_caches[layer_name]
if cache.ndim == 0:
raise ValueError(f"KV cache {layer_name!r} must be a tensor")
block_stride = cache.stride(0) * cache.element_size()
addresses[layer_name] = cache.data_ptr()
for region in self.regions:
if region.layer_name != layer_name:
continue
if (
block_stride > region.block_stride
or region.block_stride % block_stride != 0
or (
region.block_bytes > block_stride
and region.block_bytes % block_stride != 0
)
):
raise ValueError(
f"KV block stride mismatch for {layer_name!r}: "
f"config={region.block_stride}, tensor={block_stride}"
)
self._physical_strides[layer_name] = block_stride
self._base_addresses = addresses
def plan_registered_block(
self,
key: str,
block_id: int,
*,
group_id: int | None = None,
request_id: str | None = None,
) -> BlockTransferPlan:
if not self._base_addresses:
raise RuntimeError("KV caches must be registered before planning")
return self.plan_for_block(
key,
block_id,
self._base_addresses,
group_id=group_id,
request_id=request_id,
)
def plan_for_block(
self,
key: str,
block_id: int,
base_addresses: dict[str, int],
*,
group_id: int | None = None,
request_id: str | None = None,
) -> BlockTransferPlan:
"""Lower one GPU block into a deterministic scatter/gather plan."""
return BlockTransferPlan(
key=key,
block_id=block_id,
ranges=self._ranges_for_block(block_id, base_addresses, group_id),
request_id=request_id,
)
def materialize(self, plan: BlockTransferPlan) -> BlockTransferPlan:
if not self._base_addresses:
raise RuntimeError("KV caches must be registered before planning")
return replace(
plan,
ranges=self._ranges_for_block(
plan.block_id, self._base_addresses, plan.group_id
),
)
def _ranges_for_block(
self,
block_id: int,
base_addresses: dict[str, int],
group_id: int | None,
) -> tuple[KVRange, ...]:
if block_id < 0:
raise ValueError("block_id must be non-negative")
regions = (
self.regions
if group_id is None
else self._regions_by_group.get(group_id, ())
)
if group_id is not None and not regions:
raise ValueError(f"KV layout does not contain cache group {group_id}")
ranges: list[KVRange] = []
object_offset = 0
for region in regions:
try:
base_address = base_addresses[region.layer_name]
except KeyError as exc:
raise ValueError(
f"missing base address for layer {region.layer_name!r}"
) from exc
physical_stride = self._physical_strides.get(
region.layer_name, region.block_stride
)
if region.block_bytes <= physical_stride:
ranges.append(
KVRange(
layer_name=region.layer_name,
group_id=region.group_id,
block_id=block_id,
base_address=base_address + block_id * physical_stride,
stride=physical_stride,
length=region.block_bytes,
object_offset=(
object_offset
if group_id is not None
else region.object_offset
),
)
)
object_offset += region.block_bytes
continue
physical_parts = max(1, region.block_bytes // physical_stride)
for part in range(physical_parts):
part_start = part * physical_stride
part_end = min(part_start + physical_stride, region.block_bytes)
physical_block_id = block_id * physical_parts + part
ranges.append(
KVRange(
layer_name=region.layer_name,
group_id=region.group_id,
block_id=block_id,
base_address=base_address + physical_block_id * physical_stride,
stride=physical_stride,
length=part_end - part_start,
object_offset=(
object_offset + part_start
if group_id is not None
else region.object_offset + part * physical_stride
),
)
)
object_offset += region.block_bytes
return tuple(ranges)
class KVRange(NamedTuple):
"""A source/destination range in a runtime transfer."""
layer_name: str
group_id: int
block_id: int
base_address: int
stride: int
length: int
object_offset: int
@dataclass(frozen=True)
class BlockTransferPlan:
"""One complete logical block transfer."""
key: str
block_id: int
ranges: tuple[KVRange, ...] = ()
request_id: str | None = None
group_id: int | None = None
@dataclass
class LoadSpec:
"""Scheduler decision for one externally loaded prefix."""
local_tokens: int
external_tokens: int
@property
def num_tokens_to_load(self) -> int:
return max(self.external_tokens - self.local_tokens, 0)
@dataclass
class RequestTracker:
"""Core request state shared by every runtime mode."""
saved_tokens: int = 0
block_ids: tuple[list[int], ...] = ()
load_spec: LoadSpec | None = None
def reset(self) -> None:
self.saved_tokens = 0
self.block_ids = ()
self.load_spec = None
def update_blocks(self, block_ids: tuple[list[int], ...]) -> None:
if not self.block_ids:
self.block_ids = tuple(list(group) for group in block_ids)
return
if len(self.block_ids) != len(block_ids):
raise ValueError("KV cache group count changed for a request")
for current, new in zip(self.block_ids, block_ids, strict=True):
if new[: len(current)] == current:
current.extend(new[len(current) :])
elif not new or current[-len(new) :] == new:
continue
else:
current.extend(new)
def replace_blocks(self, block_ids: tuple[list[int], ...]) -> None:
"""Replace stale pre-preemption tables with resumed full tables."""
self.block_ids = tuple(list(group) for group in block_ids)
self.saved_tokens = 0
def mark_saved(self, token_count: int, block_size: int) -> int:
"""Advance the save watermark only to a complete block boundary."""
complete_tokens = token_count // block_size * block_size
if complete_tokens < self.saved_tokens:
return self.saved_tokens
self.saved_tokens = complete_tokens
return complete_tokens
class TransferJobStatus(str, Enum):
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
@dataclass
class TransferJobState:
"""Per-key completion state; one failed key does not fail its siblings."""
plans: tuple[BlockTransferPlan, ...]
status: TransferJobStatus = TransferJobStatus.PENDING
completed_keys: set[str] = field(default_factory=set)
failed_keys: set[str] = field(default_factory=set)
error: str | None = None
@property
def keys(self) -> Sequence[str]:
return tuple(plan.key for plan in self.plans)
def start(self) -> None:
if self.status != TransferJobStatus.PENDING:
raise RuntimeError(f"cannot start job in state {self.status}")
self.status = TransferJobStatus.RUNNING
def complete(self, keys: Sequence[str] = ()) -> None:
self.completed_keys.update(keys or self.keys)
self._finish_if_done()
def fail(self, keys: Sequence[str], error: str | None = None) -> None:
self.failed_keys.update(keys)
self.error = error
self._finish_if_done()
@property
def failed_block_ids(self) -> set[int]:
return {plan.block_id for plan in self.plans if plan.key in self.failed_keys}
def _finish_if_done(self) -> None:
keys = set(self.keys)
if self.completed_keys | self.failed_keys >= keys:
self.status = (
TransferJobStatus.FAILED
if self.failed_keys
else TransferJobStatus.COMPLETED
)
@dataclass
class UMBPConnectorMetadata(KVConnectorMetadata):
"""Scheduler-to-worker metadata for one engine step."""
async_load: bool = False
store_plans: list[BlockTransferPlan] = field(default_factory=list)
load_requests: dict[str, list[BlockTransferPlan]] = field(default_factory=dict)
store_requests: dict[str, list[BlockTransferPlan]] = field(default_factory=dict)
preempted_request_ids: set[str] = field(default_factory=set)
store_event: int = -1
@dataclass
class StoreEventResult:
"""Number of workers that finished one store event."""
completed_workers: int = 0
def merge(self, other: StoreEventResult) -> None:
self.completed_workers += other.completed_workers
@dataclass
class UMBPConnectorWorkerMetadata(KVConnectorWorkerMetadata):
"""Worker completion metadata aggregated across ranks."""
store_events: dict[int, StoreEventResult] = field(default_factory=dict)
def aggregate(
self, other: KVConnectorWorkerMetadata
) -> UMBPConnectorWorkerMetadata:
if not isinstance(other, UMBPConnectorWorkerMetadata):
raise TypeError("cannot aggregate incompatible UMBP worker metadata")
for event, result in other.store_events.items():
self.store_events.setdefault(event, StoreEventResult()).merge(result)
return self
@@ -1,13 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .base import IUMBPRuntime, UMBPSchedulerHandle, UMBPWorkerHandle
from .factory import UMBPRuntimeConfig, UMBPRuntimeFactory
__all__ = [
"IUMBPRuntime",
"UMBPSchedulerHandle",
"UMBPWorkerHandle",
"UMBPRuntimeConfig",
"UMBPRuntimeFactory",
]
@@ -1,69 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backend-neutral runtime contract for UMBP."""
from __future__ import annotations
from collections.abc import Sequence
from typing import Protocol
import torch
from ..data import (
BlockTransferPlan,
KVLayoutDescriptor,
RankTopology,
TransferJobState,
)
class UMBPSchedulerHandle(Protocol):
def lookup(self, keys: Sequence[str]) -> Sequence[bool]:
"""Return authoritative existence for each key."""
def clear(self) -> bool:
"""Clear all objects visible to this scheduler handle."""
def close(self) -> None:
"""Release scheduler-side resources."""
class UMBPWorkerHandle(Protocol):
def register_buffers(self, kv_caches: dict[str, torch.Tensor]) -> None:
"""Register each unique KV allocation with the runtime."""
def load(self, plans: Sequence[BlockTransferPlan]) -> TransferJobState:
"""Start loading the requested plans."""
def store(self, plans: Sequence[BlockTransferPlan]) -> TransferJobState:
"""Start storing the requested plans."""
def wait(self, job: TransferJobState) -> TransferJobState:
"""Block until the job is final and its buffers are safe to reuse."""
def poll(self, job: TransferJobState) -> TransferJobState | None:
"""Return a finished job without blocking, or None if still pending."""
def publish(self, job: TransferJobState) -> None:
"""Accept a completed store; a runtime may defer visibility until then."""
def close(self) -> None:
"""Release worker-side resources."""
class IUMBPRuntime(Protocol):
def create_scheduler_handle(
self,
namespace: str,
topology: RankTopology,
layout: KVLayoutDescriptor,
) -> UMBPSchedulerHandle:
"""Create a scheduler-side handle."""
def create_worker_handle(
self,
namespace: str,
topology: RankTopology,
layout: KVLayoutDescriptor,
) -> UMBPWorkerHandle:
"""Create a worker-side handle."""
@@ -1,76 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Runtime configuration and adapter factory for UMBP."""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass, replace
from typing import Any
from vllm.config import VllmConfig
from .base import IUMBPRuntime
@dataclass(frozen=True)
class UMBPRuntimeConfig:
"""Shared connector options passed to the selected runtime adapter."""
mode: str
options: dict[str, Any]
rank_count: int = 1
dp_index: int = 0
@classmethod
def from_vllm(cls, vllm_config: VllmConfig) -> UMBPRuntimeConfig:
transfer_config = vllm_config.kv_transfer_config
if transfer_config is None:
raise ValueError("UMBP requires kv_transfer_config")
options = dict(transfer_config.kv_connector_extra_config)
mode = options.get("mode", "embedded")
if mode not in {"embedded"}:
raise ValueError(f"unknown UMBP mode: {mode!r}")
namespace = options.get("key_namespace", "auto")
if namespace != "auto" and (not isinstance(namespace, str) or not namespace):
raise ValueError("key_namespace must be a non-empty string or 'auto'")
return cls(
mode=mode,
options=options,
dp_index=int(
getattr(vllm_config.parallel_config, "data_parallel_index", 0) or 0
),
)
def resolve_for_rank_count(self, rank_count: int) -> UMBPRuntimeConfig:
"""Pass topology to the adapter without interpreting its capacity options."""
if rank_count <= 0:
raise ValueError("rank_count must be positive")
return replace(self, rank_count=rank_count)
RuntimeBuilder = Callable[[UMBPRuntimeConfig], IUMBPRuntime]
class UMBPRuntimeFactory:
"""Registry used by UMBP adapters without importing MORI in vLLM."""
_builders: dict[str, RuntimeBuilder] = {}
@classmethod
def register(cls, mode: str, builder: RuntimeBuilder) -> None:
if mode not in {"embedded"}:
raise ValueError(f"unknown UMBP mode: {mode!r}")
if mode in cls._builders:
raise ValueError(f"UMBP runtime mode {mode!r} is already registered")
cls._builders[mode] = builder
@classmethod
def build(cls, config: UMBPRuntimeConfig) -> IUMBPRuntime:
try:
builder = cls._builders[config.mode]
except KeyError as exc:
raise RuntimeError(
f"no UMBP runtime adapter is registered for mode {config.mode!r}"
) from exc
return builder(config)
@@ -1,518 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared scheduler-side UMBP connector behavior."""
from __future__ import annotations
import logging
from collections.abc import Sequence
from dataclasses import dataclass, field
from functools import lru_cache
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.logger import init_logger
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import (
resolve_dcp_kv_block_size,
resolve_kv_cache_block_sizes,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
from .data import (
BlockIdentityCodec,
BlockTransferPlan,
LoadSpec,
RankTopology,
RequestTracker,
StoreEventResult,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
)
from .runtime import UMBPSchedulerHandle
logger = init_logger(__name__)
@lru_cache(maxsize=8192)
def _rank_keys(
codec: BlockIdentityCodec,
topology: RankTopology,
block_hash: bytes,
group_id: int,
) -> tuple[str, ...]:
"""Reuse immutable object identities, not runtime residency."""
return codec.keys_for_topology(block_hash, topology, (group_id,))
@dataclass
class _StoreEvent:
plans: tuple[BlockTransferPlan, ...]
result: StoreEventResult = field(default_factory=StoreEventResult)
class UMBPStoreConnectorScheduler:
"""Own vLLM prefix matching while the runtime owns lookup transport."""
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
runtime: UMBPSchedulerHandle,
codec: BlockIdentityCodec,
topology: RankTopology | None = None,
) -> None:
self.block_size, resolved_hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self.kv_cache_config = kv_cache_config
dcp_size = vllm_config.parallel_config.decode_context_parallel_size
self.group_block_sizes = {
group_id: resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_size)
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
}
self.runtime = runtime
self.codec = codec
self.topology = topology or RankTopology()
transfer_config = vllm_config.kv_transfer_config
assert transfer_config is not None
extra = transfer_config.kv_connector_extra_config
self.load_async = bool(extra.get("load_async", True))
self.enable_lookup = bool(extra.get("enable_lookup", True))
self.save_decode_cache = bool(extra.get("save_decode_cache", False))
# A kv_consumer only restores from the pool, as with other store
# connectors; it never offloads its own KV.
self.store_enabled = getattr(transfer_config, "kv_role", None) != "kv_consumer"
# Request.block_hashes are computed at this size; keys index into them.
self.hash_block_size = resolved_hash_block_size
speculative_config = getattr(vllm_config, "speculative_config", None)
self.use_eagle = bool(
speculative_config is not None and speculative_config.use_eagle_block_drop()
)
self._pending_loads: dict[str, list[BlockTransferPlan]] = {}
self._load_specs: dict[str, LoadSpec] = {}
self._lookup_units = {
group_id: self.group_block_sizes[group_id]
for group_id in kv_cache_config.prefix_cacheable_group_ids
}
self._requests: dict[str, Request] = {}
self._request_trackers: dict[str, RequestTracker] = {}
self._gpu_block_pool: BlockPool | None = None
self._store_events: dict[int, _StoreEvent] = {}
self._num_workers = getattr(vllm_config.parallel_config, "world_size", 1)
self._store_event_counter = 0
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
self._gpu_block_pool = gpu_block_pool
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
if not self.enable_lookup:
return 0, False
hashes = request.block_hashes
if not hashes or request.num_tokens < self.block_size:
return 0, False
if num_computed_tokens % self.block_size != 0:
return 0, False
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
# vLLM must execute at least the final prompt token. Returning a hit
# for the entire prompt leaves the scheduler with no new token to run.
max_external_token = request.num_tokens - 1
if max_external_token <= num_computed_tokens:
self._load_specs.pop(request.request_id, None)
return 0, False
logical_objects_by_group, keys = self._lookup_objects(
request, num_computed_tokens, hashes
)
try:
hits = list(self.runtime.lookup(keys))
except Exception as exc:
logger.debug("UMBP lookup failed request=%s: %s", request.request_id, exc)
return 0, False
if len(hits) != len(keys):
logger.debug(
"UMBP lookup returned an invalid result length request=%s",
request.request_id,
)
return 0, False
per_rank = self.topology.rank_count
offset = 0
logical_hits_by_group: dict[int, list[bool]] = {}
for group_id in group_ids:
group_size = len(logical_objects_by_group[group_id]) * per_rank
group_hits = hits[offset : offset + group_size]
offset += group_size
logical_hits_by_group[group_id] = (
group_hits
if per_rank == 1
else [
all(group_hits[index : index + per_rank])
for index in range(0, len(group_hits), per_rank)
]
)
group_id = group_ids[0]
matched_blocks = 0
for hit in logical_hits_by_group[group_id]:
if not hit:
break
matched_blocks += 1
if self.use_eagle:
# As core's coordinator: the drafter recomputes the last block.
matched_blocks = max(0, matched_blocks - 1)
need_to_load = matched_blocks * self._lookup_units[group_id]
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"UMBP lookup result request=%s keys=%s group_hits=%s load_tokens=%d",
request.request_id,
keys,
logical_hits_by_group,
need_to_load,
)
matched_tokens = num_computed_tokens + need_to_load
if need_to_load <= 0:
return 0, False
self._load_specs[request.request_id] = LoadSpec(
local_tokens=num_computed_tokens,
external_tokens=matched_tokens,
)
return need_to_load, self.load_async
def _lookup_objects(
self, request: Request, local_tokens: int, hashes: Sequence[bytes]
) -> tuple[dict[int, list[tuple[bytes, tuple[str, ...]]]], list[str]]:
objects: dict[int, list[tuple[bytes, tuple[str, ...]]]] = {}
keys: list[str] = []
for group_id, unit in self._lookup_units.items():
group_objects = []
for token_end in range(local_tokens + unit, request.num_tokens, unit):
block_hash = self._object_hash_at_token_end(hashes, token_end)
rank_keys = _rank_keys(self.codec, self.topology, block_hash, group_id)
group_objects.append((block_hash, rank_keys))
keys.extend(rank_keys)
objects[group_id] = group_objects
return objects, keys
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
) -> None:
self._requests[request.request_id] = request
block_groups = blocks.get_block_ids(
group_ids=self.kv_cache_config.prefix_cacheable_group_ids
)
if not block_groups:
return
tracker = self._tracker_for_request(request.request_id)
# Core passes the request's whole table on every admission. A request
# whose load failed is recomputed into newly allocated blocks.
tracker.replace_blocks(tuple(list(group) for group in block_groups))
if num_external_tokens <= 0:
self._pending_loads.pop(request.request_id, None)
self._load_specs.pop(request.request_id, None)
return
spec = self._load_specs.get(request.request_id)
if spec is not None:
tracker.load_spec = spec
# A request admitted without its load would wait for it forever.
assert num_external_tokens == spec.num_tokens_to_load, (
f"UMBP matched {spec.num_tokens_to_load} external tokens for "
f"request {request.request_id} but {num_external_tokens} were "
"allocated"
)
plans = self._load_plans_for_external_tokens(
request,
tracker,
block_groups,
num_external_tokens,
)
self._pending_loads[request.request_id] = plans
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
meta = UMBPConnectorMetadata(async_load=self.load_async)
for request_id in scheduler_output.finished_req_ids:
self._pending_loads.pop(request_id, None)
self._load_specs.pop(request_id, None)
self._request_trackers.pop(request_id, None)
self._requests.pop(request_id, None)
for request in scheduler_output.scheduled_new_reqs:
load_plans = self._pending_loads.pop(request.req_id, [])
self._load_specs.pop(request.req_id, None)
if load_plans:
meta.load_requests[request.req_id] = load_plans
else:
tracker = self._tracker_for_request(request.req_id)
total_tokens = (
request.num_computed_tokens
+ scheduler_output.num_scheduled_tokens[request.req_id]
)
tracked_request = self._requests.get(request.req_id)
selected_block_ids = tuple(
request.block_ids[group_id]
for group_id in (self.kv_cache_config.prefix_cacheable_group_ids)
)
store_plans = (
[]
if not self.store_enabled or tracked_request is None
else self._store_plans(
tracked_request,
tracker,
total_tokens,
block_ids_override=selected_block_ids,
)
)
logger.debug(
"UMBP store planning request=%s tokens=%d plans=%d keys=%s",
request.req_id,
total_tokens,
len(store_plans),
tuple(plan.key for plan in store_plans),
)
meta.store_plans.extend(store_plans)
if store_plans:
meta.store_requests[request.req_id] = store_plans
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
cached_reqs = scheduler_output.scheduled_cached_reqs
resumed_ids: set[str] = set(getattr(cached_reqs, "resumed_req_ids", ()))
for request_id in cached_reqs.req_ids:
load_plans = self._pending_loads.pop(request_id, [])
self._load_specs.pop(request_id, None)
if load_plans:
meta.load_requests[request_id] = load_plans
elif self.store_enabled:
cached_request = self._requests.get(request_id)
cached_tracker = self._request_trackers.get(request_id)
if cached_request is not None and cached_tracker is not None:
request_index = cached_reqs.req_ids.index(request_id)
num_computed_tokens = cached_reqs.num_computed_tokens[request_index]
# num_tokens counts sampled tokens, so a decode step is still
# short of it; only the prompt bounds the prefill.
is_prefill = num_computed_tokens < cached_request.num_prompt_tokens
if not is_prefill and not self.save_decode_cache:
continue
new_block_ids = cached_reqs.new_block_ids[request_index]
if new_block_ids:
selected = tuple(
new_block_ids[group_id]
for group_id in (
self.kv_cache_config.prefix_cacheable_group_ids
)
)
if request_id in resumed_ids:
cached_tracker.replace_blocks(selected)
else:
cached_tracker.update_blocks(selected)
# The appended table keeps blocks core has since freed, such
# as sliding-window blocks out of the window, which a store
# would read; core's current table does not.
current = (
block_state.get_block_ids(request_id)
if block_state is not None
else None
)
if current is not None:
cached_tracker.block_ids = tuple(
list(current[group_id])
for group_id in (
self.kv_cache_config.prefix_cacheable_group_ids
)
)
total_tokens = (
num_computed_tokens
+ scheduler_output.num_scheduled_tokens[request_id]
)
if not self.save_decode_cache:
total_tokens = min(
total_tokens, cached_request.num_prompt_tokens
)
store_plans = self._store_plans(
cached_request,
cached_tracker,
total_tokens,
block_ids_override=cached_tracker.block_ids,
)
meta.store_plans.extend(store_plans)
if store_plans:
meta.store_requests[request_id] = store_plans
# Async loads are admitted without scheduling model tokens, so those
# requests are absent from both scheduled request collections above.
# Forward their plans explicitly so workers can start the transfer and
# eventually move the request out of WAITING_FOR_REMOTE_KVS.
for request_id, load_plans in list(self._pending_loads.items()):
self._pending_loads.pop(request_id, None)
self._load_specs.pop(request_id, None)
if load_plans:
meta.load_requests[request_id] = load_plans
for request_id in scheduler_output.preempted_req_ids or set():
meta.preempted_request_ids.add(request_id)
self._load_specs.pop(request_id, None)
if preempted_tracker := self._request_trackers.get(request_id):
preempted_tracker.reset()
event_plans = tuple(meta.store_plans)
if event_plans:
event = self._store_event_counter
self._store_event_counter += 1
meta.store_event = event
self._store_events[event] = _StoreEvent(event_plans)
if self._gpu_block_pool is not None:
self._gpu_block_pool.touch(
[
self._gpu_block_pool.blocks[block_id]
for block_id in dict.fromkeys(
plan.block_id for plan in event_plans
)
]
)
return meta
def _tracker_for_request(self, request_id: str) -> RequestTracker:
return self._request_trackers.setdefault(request_id, RequestTracker())
def _load_plans_for_external_tokens(
self,
request: Any,
tracker: RequestTracker,
block_groups: tuple[list[int], ...],
num_external_tokens: int,
) -> list[BlockTransferPlan]:
"""Plan the logical objects selected by the external hit window."""
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
spec = tracker.load_spec
local_tokens = spec.local_tokens if spec is not None else 0
end_tokens = local_tokens + num_external_tokens
plans: list[BlockTransferPlan] = []
for group_index, group_id in enumerate(group_ids):
group_block_size = self.group_block_sizes[group_id]
start_block = local_tokens // group_block_size
num_full_blocks = end_tokens // group_block_size
for block_index in range(start_block, num_full_blocks):
token_end = (block_index + 1) * group_block_size
block_hash = self._object_hash_at_token_end(
request.block_hashes, token_end
)
plans.append(
BlockTransferPlan(
key=self.codec.key(block_hash, group_id=group_id),
block_id=block_groups[group_index][block_index],
request_id=request.request_id,
group_id=group_id,
)
)
return plans
def _object_hash_at_token_end(
self, hashes: Sequence[bytes], token_end: int
) -> bytes:
hash_index = token_end // self.hash_block_size - 1
if hash_index < 0 or hash_index >= len(hashes):
raise ValueError(
f"request has {len(hashes)} hashes, needs index {hash_index}"
)
return hashes[hash_index]
def _store_plans(
self,
request: Any,
tracker: RequestTracker,
token_count: int,
block_ids_override: tuple[list[int], ...] | None = None,
) -> list[BlockTransferPlan]:
"""Describe full blocks produced by a scheduled prefill."""
request_id = getattr(request, "request_id", None) or request.req_id
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
# Scheduled speculative tokens are not yet verified and have no block
# hash; a block is storable only once its hash exists.
token_count = min(token_count, len(request.block_hashes) * self.hash_block_size)
previous_saved_tokens = tracker.saved_tokens
# Hash-aligned rather than block-aligned: a group with smaller blocks
# than the largest must not wait for that group's block to fill.
save_to = tracker.mark_saved(token_count, self.hash_block_size)
plans: list[BlockTransferPlan] = []
block_groups = (
block_ids_override
if block_ids_override is not None
else tuple(request.block_ids[group_id] for group_id in group_ids)
)
block_ranges = {
group_id: range(
previous_saved_tokens // self.group_block_sizes[group_id],
min(save_to // self.group_block_sizes[group_id], len(block_ids)),
)
for group_id, block_ids in zip(group_ids, block_groups)
}
for group_id, block_ids in zip(group_ids, block_groups):
if group_id not in block_ranges:
continue
group_block_size = self.group_block_sizes[group_id]
for index in block_ranges[group_id]:
if block_ids[index] == NULL_BLOCK_ID:
continue
token_end = (index + 1) * group_block_size
block_hash = self._object_hash_at_token_end(
request.block_hashes, token_end
)
plans.append(
BlockTransferPlan(
key=self.codec.key(block_hash, group_id=group_id),
block_id=block_ids[index],
request_id=request_id,
group_id=group_id,
)
)
return plans
def request_finished(
self, request: Request, block_ids: tuple[list[int], ...]
) -> tuple[bool, dict[str, Any] | None]:
self._pending_loads.pop(request.request_id, None)
self._load_specs.pop(request.request_id, None)
self._requests.pop(request.request_id, None)
return False, None
def update_connector_output(self, output: KVConnectorOutput) -> None:
metadata = output.kv_connector_worker_meta
if not isinstance(metadata, UMBPConnectorWorkerMetadata):
return
pool = self._gpu_block_pool
for event_id, result in metadata.store_events.items():
event = self._store_events.get(event_id)
if event is None:
continue
event.result.merge(result)
if event.result.completed_workers < self._num_workers:
continue
del self._store_events[event_id]
if pool is not None:
block_ids = dict.fromkeys(plan.block_id for plan in event.plans)
pool.free_blocks(
pool.blocks[block_id] for block_id in reversed(block_ids)
)
def has_pending_push_work(self) -> bool:
return bool(self._store_events)
def reset_store(self) -> bool:
if self._store_events:
return False
self._load_specs.clear()
self._pending_loads.clear()
return self.runtime.clear()
def close(self) -> None:
self.runtime.close()
@@ -1,240 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared worker-side UMBP connector behavior."""
from __future__ import annotations
from dataclasses import dataclass, replace
import torch
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from .data import (
BlockIdentityCodec,
BlockTransferPlan,
KVLayoutPlanner,
StoreEventResult,
TransferJobState,
TransferJobStatus,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
)
from .runtime import UMBPWorkerHandle
logger = init_logger(__name__)
@dataclass
class _StoreBatch:
event_id: int
jobs: dict[str, TransferJobState]
class UMBPStoreConnectorWorker:
"""Translate shared metadata into runtime load/store jobs."""
def __init__(
self,
runtime: UMBPWorkerHandle,
layout: KVLayoutPlanner | None = None,
*,
codec: BlockIdentityCodec | None = None,
) -> None:
self.runtime = runtime
self.layout = layout
self.codec = codec
self._load_jobs: dict[str, dict[str | None, TransferJobState]] = {}
self._store_jobs: dict[str, TransferJobState] = {}
self._pending_stores: list[_StoreBatch] = []
self._worker_meta = UMBPConnectorWorkerMetadata()
self._finished_recving: set[str] = set()
self._load_error_block_ids: set[int] = set()
self._report_load_completions = False
self._active_store_event = -1
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
if self.layout is not None and self.layout.regions:
self.layout.register_kv_caches(kv_caches)
self.runtime.register_buffers(kv_caches)
def _materialize_plans(
self, plans: list[BlockTransferPlan]
) -> list[BlockTransferPlan]:
plans = [self._localize_plan(plan) for plan in plans]
if self.layout is None or not self.layout.regions:
return plans
materialized: list[BlockTransferPlan] = []
for plan in plans:
if plan.ranges:
materialized.append(plan)
continue
materialized.append(self.layout.materialize(plan))
return materialized
def _localize_plan(self, plan: BlockTransferPlan) -> BlockTransferPlan:
if self.codec is None or plan.group_id is None:
return plan
key = self._localize_key(plan.key, plan.group_id)
if key == plan.key:
return plan
return replace(plan, key=key)
def _localize_key(self, key: str, group_id: int) -> str:
if self.codec is None:
return key
try:
block_hash = bytes.fromhex(key.rsplit(":", 1)[1])
except (IndexError, ValueError):
return key
return self.codec.key(block_hash, group_id)
def start_load_kv(
self, forward_context: ForwardContext, metadata: UMBPConnectorMetadata
) -> None:
del forward_context
self._report_load_completions = metadata.async_load
for request_id, plans in metadata.load_requests.items():
if plans:
self._submit_loads(request_id, plans)
def _submit_loads(self, request_id: str, plans: list[BlockTransferPlan]) -> None:
materialized = self._materialize_plans(plans)
logger.debug(
"UMBP load submitted request=%s plans=%d ranges=%d bytes=%d",
request_id,
len(materialized),
sum(len(plan.ranges) for plan in materialized),
sum(item.length for plan in materialized for item in plan.ranges),
)
self._load_jobs[request_id] = {None: self.runtime.load(materialized)}
def _finish_job(
self, request_id: str, result: TransferJobState, *, is_load: bool
) -> bool:
operation = "load" if is_load else "store"
succeeded = result.status == TransferJobStatus.COMPLETED
if not succeeded:
logger.debug(
"UMBP %s failed request=%s status=%s error=%s",
operation,
request_id,
result.status.value,
result.error,
)
if is_load:
self._load_error_block_ids.update(result.failed_block_ids)
elif succeeded:
self.runtime.publish(result)
return succeeded
def wait_for_layer_load(self, layer_name: str) -> None:
if self._report_load_completions:
# Asynchronous loads fill blocks of requests that do not run until
# get_finished reports them, so no forward pass reads those blocks.
return
self._drain_load_jobs(wait=True)
def wait_for_save(self) -> None:
if self._store_jobs or self._active_store_event >= 0:
self._pending_stores.append(
_StoreBatch(self._active_store_event, self._store_jobs)
)
self._store_jobs = {}
self._active_store_event = -1
self._drain_store_jobs(wait=False)
def _drain_store_jobs(self, *, wait: bool) -> None:
for batch in list(self._pending_stores):
for request_id, job in list(batch.jobs.items()):
result = self.runtime.wait(job) if wait else self.runtime.poll(job)
if result is None:
continue
# A partly failed job is not published, including its stored keys.
self._finish_job(request_id, result, is_load=False)
del batch.jobs[request_id]
if not batch.jobs:
if batch.event_id >= 0:
self._worker_meta.store_events[batch.event_id] = StoreEventResult(
completed_workers=1
)
self._pending_stores.remove(batch)
def _submit_store_plans(
self, plans: list[BlockTransferPlan], request_id: str
) -> None:
if not plans:
return
materialized = self._materialize_plans(plans)
self._store_jobs[request_id] = self.runtime.store(materialized)
def enqueue_stores(self, metadata: UMBPConnectorMetadata) -> None:
self._active_store_event = metadata.store_event
store_requests = {
request_id: list(plans)
for request_id, plans in metadata.store_requests.items()
}
grouped = {plan for plans in store_requests.values() for plan in plans}
for plan in metadata.store_plans:
if plan not in grouped:
store_requests.setdefault(
plan.request_id or "__umbp_batch__", []
).append(plan)
grouped.add(plan)
for request_id, plans in store_requests.items():
self._submit_store_plans(plans, request_id)
def handle_preemptions(self, metadata: UMBPConnectorMetadata) -> None:
"""Finish request-local jobs before vLLM reuses their GPU blocks."""
preempted = metadata.preempted_request_ids
if not preempted:
return
for request_id in preempted:
for job in self._load_jobs.pop(request_id, {}).values():
self.runtime.wait(job)
for jobs in [self._store_jobs, *(batch.jobs for batch in self._pending_stores)]:
for request_id, job in jobs.items():
if request_id in preempted or any(
plan.request_id in preempted for plan in job.plans
):
jobs[request_id] = self.runtime.wait(job)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
del finished_req_ids
self._drain_load_jobs(wait=False)
self._drain_store_jobs(wait=False)
recving = set(self._finished_recving)
self._finished_recving.difference_update(recving)
return None, recving or None
def _drain_load_jobs(self, *, wait: bool) -> None:
for request_id, jobs in list(self._load_jobs.items()):
for layer, job in list(jobs.items()):
result = self.runtime.wait(job) if wait else self.runtime.poll(job)
if result is None:
continue
del jobs[layer]
self._finish_job(request_id, result, is_load=True)
if not jobs:
del self._load_jobs[request_id]
if self._report_load_completions:
self._finished_recving.add(request_id)
def build_connector_worker_meta(self) -> UMBPConnectorWorkerMetadata:
result = self._worker_meta
self._worker_meta = UMBPConnectorWorkerMetadata()
return result
def get_block_ids_with_load_errors(self) -> set[int]:
failed = self._load_error_block_ids
self._load_error_block_ids = set()
return failed
def close(self) -> None:
self.wait_for_save()
self._drain_store_jobs(wait=True)
self._drain_load_jobs(wait=True)
self.runtime.close()
-144
View File
@@ -1,144 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared configuration and setup for EC and KV Mooncake Store clients."""
import json
import os
from dataclasses import dataclass
from typing import Any, Literal
import regex as re
DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
DEFAULT_LOCAL_BUFFER_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
DEFAULT_TENANT_ID = "default"
MooncakeMode = Literal["embedded", "standalone-store"]
@dataclass
class MooncakeStoreConfig:
"""Configuration for MooncakeDistributedStore.
``mode`` selects the topology: ``embedded`` (each rank contributes
``global_segment_size`` in-process) or ``standalone-store`` (rank
contributes 0; an external ``mooncake_client`` process owns the pool
and the SSD tier).
"""
metadata_server: str
master_server_address: str
protocol: str
device_name: str
mode: MooncakeMode = "embedded"
global_segment_size: int = DEFAULT_GLOBAL_SEGMENT_SIZE
local_buffer_size: int = DEFAULT_LOCAL_BUFFER_SIZE
enable_offload: bool = False
tenant_id: str = DEFAULT_TENANT_ID
def __post_init__(self) -> None:
if self.mode not in ("embedded", "standalone-store"):
raise ValueError(f"unknown Mooncake mode: {self.mode!r}")
if self.local_buffer_size <= 0:
raise ValueError("local_buffer_size must be > 0")
if self.mode == "embedded" and self.global_segment_size == 0:
raise ValueError("embedded mode requires global_segment_size > 0")
if self.mode == "standalone-store" and self.global_segment_size != 0:
raise ValueError("standalone-store mode requires global_segment_size == 0")
@staticmethod
def from_file(file_path: str) -> "MooncakeStoreConfig":
with open(file_path) as file:
config = json.load(file)
return MooncakeStoreConfig(
metadata_server=config.get("metadata_server", ""),
master_server_address=config.get("master_server_address", ""),
protocol=config.get("protocol", "rdma"),
device_name=config.get("device_name", ""),
mode=config.get("mode", "embedded"),
global_segment_size=_parse_size(
config.get("global_segment_size", DEFAULT_GLOBAL_SEGMENT_SIZE)
),
local_buffer_size=_parse_size(
config.get("local_buffer_size", DEFAULT_LOCAL_BUFFER_SIZE)
),
enable_offload=bool(config.get("enable_offload", False)),
tenant_id=_normalize_tenant_id(config.get("tenant_id", DEFAULT_TENANT_ID)),
)
@staticmethod
def load_from_config() -> "MooncakeStoreConfig":
config_path = os.getenv("MOONCAKE_CONFIG_PATH")
if not config_path:
raise ValueError(
"The environment variable 'MOONCAKE_CONFIG_PATH' is not set."
)
return MooncakeStoreConfig.from_file(config_path)
def _normalize_tenant_id(value: Any) -> str:
if value is None:
return DEFAULT_TENANT_ID
if not isinstance(value, str):
raise TypeError(
f"tenant_id must be a string or null, got {type(value).__name__}: {value!r}"
)
tenant_id = value.strip()
return tenant_id if tenant_id else DEFAULT_TENANT_ID
def _parse_size(value: Any) -> int:
"""Parse storage size strings with units: GB, MB, KB, B."""
if isinstance(value, int):
return value
if not isinstance(value, str):
try:
return int(value)
except (TypeError, ValueError) as e:
raise TypeError(f"Unsupported type for size: {type(value)}") from e
cleaned = value.strip().lower()
if not cleaned:
raise ValueError("Size cannot be empty.")
unit_multipliers = {
"gb": 1024**3,
"mb": 1024**2,
"kb": 1024,
"b": 1,
}
match = re.match(r"^\s*([\d.]+)\s*(gb|mb|kb|b)?\s*$", cleaned)
if not match:
raise ValueError(f"Invalid format: '{value}'")
number_str = match.group(1)
unit = match.group(2) or "b"
multiplier = unit_multipliers[unit]
try:
numeric_value = float(number_str)
except ValueError as exc:
raise ValueError(f"Invalid numeric value '{number_str}' in: '{value}'") from exc
return int(numeric_value * multiplier)
def setup_mooncake_store(
store: Any, config: MooncakeStoreConfig, local_hostname: str
) -> None:
"""Initialize a native Store, preserving explicit tenant configuration."""
setup_kwargs: dict[str, str] = {}
if config.tenant_id != DEFAULT_TENANT_ID:
setup_kwargs["tenant_id"] = config.tenant_id
ret = store.setup(
local_hostname,
config.metadata_server,
config.global_segment_size,
config.local_buffer_size,
config.protocol,
config.device_name,
config.master_server_address,
**setup_kwargs,
)
if ret != 0:
raise RuntimeError("Initialize MooncakeDistributedStore failed.")
-27
View File
@@ -1,27 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import warnings
from vllm.entrypoints.launchers.grpc_server import main, serve_grpc
warnings.warn(
"`vllm.entrypoints.grpc_server is deprecated and will likely be "
"unsupported in a future version. Use the corresponding function from "
"`vllm.entrypoints.launchers.grpc_server` instead.",
DeprecationWarning,
stacklevel=1,
)
__all__ = ["main", "serve_grpc"]
if __name__ == "__main__":
warnings.warn(
"`python -m vllm.entrypoints.grpc_server is deprecated and "
"will likely be unsupported in a future version. "
"Please use `vllm serve <model_path> --grpc` instead.",
DeprecationWarning,
stacklevel=1,
)
main()
+4 -185
View File
@@ -9,7 +9,7 @@ import tempfile
from argparse import Namespace
from collections.abc import Awaitable, Callable
from http import HTTPStatus
from io import BytesIO, StringIO
from io import StringIO
from typing import Any, NoReturn, TypeAlias
from urllib.parse import urlparse
@@ -17,9 +17,8 @@ import aiohttp
import pybase64 as base64
import pydantic
import torch
from fastapi import UploadFile
from prometheus_client import start_http_server
from pydantic import Field, TypeAdapter, field_validator, model_validator
from pydantic import TypeAdapter, field_validator
from pydantic_core.core_schema import ValidationInfo
from starlette.datastructures import State
from starlette.responses import JSONResponse
@@ -37,19 +36,9 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
)
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.protocol import (
ErrorInfo,
ErrorResponse,
OpenAIBaseModel,
)
from vllm.entrypoints.speech_to_text.transcription.protocol import (
TranscriptionRequest,
TranscriptionResponseVariant,
)
from vllm.entrypoints.speech_to_text.translation.protocol import (
TranslationRequest,
TranslationResponse,
TranslationResponseVerbose,
)
from vllm.exceptions import VLLMValidationError
from vllm.logger import configure_logging_from_args, init_logger
from vllm.reasoning import ReasoningParserManager
@@ -76,69 +65,7 @@ def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
)
class BatchTranscriptionRequest(TranscriptionRequest):
"""Batch transcription request that uses file_url instead of file.
This class extends TranscriptionRequest but replaces the file field
with file_url to support batch processing from audio files written in JSON format.
"""
file_url: str = Field(
...,
description=(
"Either a URL of the audio or a data URL with base64 encoded audio data. "
),
)
# Override file to be optional and unused for batch processing
file: UploadFile | None = Field(default=None, exclude=True) # type: ignore[assignment]
@model_validator(mode="before")
@classmethod
def validate_no_file(cls, data: Any):
"""Ensure file field is not provided in batch requests."""
if isinstance(data, dict) and "file" in data:
raise VLLMValidationError(
"The 'file' field is not supported in batch requests. "
"Use 'file_url' instead.",
parameter="file",
)
return data
class BatchTranslationRequest(TranslationRequest):
"""Batch translation request that uses file_url instead of file.
This class extends TranslationRequest but replaces the file field
with file_url to support batch processing from audio files written in JSON format.
"""
file_url: str = Field(
...,
description=(
"Either a URL of the audio or a data URL with base64 encoded audio data. "
),
)
# Override file to be optional and unused for batch processing
file: UploadFile | None = Field(default=None, exclude=True) # type: ignore[assignment]
@model_validator(mode="before")
@classmethod
def validate_no_file(cls, data: Any):
"""Ensure file field is not provided in batch requests."""
if isinstance(data, dict) and "file" in data:
raise VLLMValidationError(
"The 'file' field is not supported in batch requests. "
"Use 'file_url' instead.",
parameter="file",
)
return data
BatchRequestInputBody: TypeAlias = (
ChatCompletionRequest | BatchTranscriptionRequest | BatchTranslationRequest
)
BatchRequestInputBody: TypeAlias = ChatCompletionRequest
class BatchRequestInput(OpenAIBaseModel):
@@ -169,18 +96,11 @@ class BatchRequestInput(OpenAIBaseModel):
url: str = info.data["url"]
if url == "/v1/chat/completions":
return ChatCompletionRequest.model_validate(value)
if url == "/v1/audio/transcriptions":
return BatchTranscriptionRequest.model_validate(value)
if url == "/v1/audio/translations":
return BatchTranslationRequest.model_validate(value)
return TypeAdapter(BatchRequestInputBody).validate_python(value)
AllResponse: TypeAlias = (
ChatCompletionResponse
| TranscriptionResponseVariant
| TranslationResponse
| TranslationResponseVerbose
)
@@ -612,78 +532,6 @@ def handle_endpoint_request(
return run_request(handler_fn, request, tracker)
def make_transcription_wrapper(
is_translation: bool,
allowed_media_domains: list[str] | None = None,
) -> WrapperFn:
"""Factory function to create a wrapper for transcription/translation handlers.
The wrapper converts BatchTranscriptionRequest or BatchTranslationRequest
to TranscriptionRequest or TranslationRequest and calls the appropriate handler.
Args:
is_translation: If True, process as translation; otherwise process
as transcription
allowed_media_domains: If set, only URLs from these domains are
permitted for HTTP/HTTPS fetches.
Returns:
A function that takes a handler and returns a wrapped handler
"""
def wrapper(handler_fn: Callable):
async def transcription_wrapper(
batch_request_body: (BatchTranscriptionRequest | BatchTranslationRequest),
) -> (
TranscriptionResponseVariant
| TranslationResponse
| TranslationResponseVerbose
| ErrorResponse
):
try:
# Download data from URL
audio_data = await download_bytes_from_url(
batch_request_body.file_url,
allowed_media_domains=allowed_media_domains,
)
# Create a mock file from the downloaded audio data
mock_file = UploadFile(
file=BytesIO(audio_data),
filename="audio.bin",
)
# Convert batch request to regular request
# by copying all fields except file_url and setting file to mock_file
request_dict = batch_request_body.model_dump(exclude={"file_url"})
request_dict["file"] = mock_file
if is_translation:
# Create TranslationRequest from BatchTranslationRequest
translation_request = TranslationRequest.model_validate(
request_dict
)
return await handler_fn(audio_data, translation_request)
else:
# Create TranscriptionRequest from BatchTranscriptionRequest
transcription_request = TranscriptionRequest.model_validate(
request_dict
)
return await handler_fn(audio_data, transcription_request)
except Exception as e:
operation = "translation" if is_translation else "transcription"
return ErrorResponse(
error=ErrorInfo(
message=f"Failed to process {operation}: {str(e)}",
type="BadRequestError",
code=HTTPStatus.BAD_REQUEST.value,
)
)
return transcription_wrapper
return wrapper
async def build_endpoint_registry(
engine_client: EngineClient,
@@ -712,10 +560,6 @@ async def build_endpoint_registry(
# Get serving objects from state (defaulting to None if not set)
openai_serving_chat = getattr(state, "openai_serving_chat", None)
openai_serving_transcription = getattr(state, "openai_serving_transcription", None)
openai_serving_translation = getattr(state, "openai_serving_translation", None)
allowed_media_domains = getattr(args, "allowed_media_domains", None)
# Registry of endpoint configurations
endpoint_registry: dict[str, dict[str, Any]] = {
@@ -728,30 +572,6 @@ async def build_endpoint_registry(
),
"wrapper_fn": None,
},
"transcriptions": {
"url_matcher": lambda url: url == "/v1/audio/transcriptions",
"handler_getter": lambda: (
openai_serving_transcription.create_transcription
if openai_serving_transcription is not None
else None
),
"wrapper_fn": make_transcription_wrapper(
is_translation=False,
allowed_media_domains=allowed_media_domains,
),
},
"translations": {
"url_matcher": lambda url: url == "/v1/audio/translations",
"handler_getter": lambda: (
openai_serving_translation.create_translation
if openai_serving_translation is not None
else None
),
"wrapper_fn": make_transcription_wrapper(
is_translation=True,
allowed_media_domains=allowed_media_domains,
),
},
}
return endpoint_registry
@@ -812,8 +632,7 @@ async def run_batch(
make_async_error_request_output(
request,
error_msg=f"URL {request.url} was used. "
"Supported endpoints: /v1/chat/completions,"
" /v1/audio/transcriptions, /v1/audio/translations.",
"Supported endpoints: /v1/chat/completions.",
)
)
@@ -46,14 +46,6 @@ def init_entrypoints_middleware(
app.add_middleware(ScalingMiddleware)
if "realtime" in supported_tasks:
# Add WebSocket metrics middleware
from vllm.entrypoints.speech_to_text.realtime.metrics import (
WebSocketMetricsMiddleware,
)
app.add_middleware(WebSocketMetricsMiddleware)
if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE:
logger.warning(
"CAUTION: Enabling log response in the API Server. "
+2 -2
View File
@@ -2,6 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Kernel implementations for vLLM."""
from . import aiter_ops, oink_ops, triton, vllm_c
from . import aiter_ops, triton, vllm_c
__all__ = ["vllm_c", "aiter_ops", "oink_ops", "triton"]
__all__ = ["vllm_c", "aiter_ops", "triton"]
-37
View File
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helion integration for vLLM."""
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import (
ConfigManager,
ConfigSet,
)
from vllm.kernels.helion.register import (
ConfigPicker,
ConfiguredHelionKernel,
HelionKernelWrapper,
get_kernel_by_name,
get_registered_kernels,
register_kernel,
vllm_helion_lib,
)
from vllm.kernels.helion.utils import canonicalize_gpu_name, get_canonical_gpu_name
__all__ = [
# Config management
"CaseKey",
"ConfigManager",
"ConfigSet",
# Kernel registration
"ConfigPicker",
"ConfiguredHelionKernel",
"HelionKernelWrapper",
"get_kernel_by_name",
"get_registered_kernels",
"register_kernel",
"vllm_helion_lib",
# Utilities
"canonicalize_gpu_name",
"get_canonical_gpu_name",
]
-70
View File
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Structured key for identifying kernel config/autotune/benchmark cases."""
from __future__ import annotations
import json
from typing import Any
class CaseKey(dict[str, Any]):
"""Immutable, hashable dict for identifying kernel cases.
Used as the key for config lookup, autotuning, benchmarking, and
input generation. Behaves like a read-only dict and can be used
as a dict key or in sets.
The canonical string form (``__str__``) is stable JSON with sorted
keys. Use ``CaseKey.default()`` for the default/fallback key.
The regular constructor requires at least one key-value pair::
CaseKey({"intermediate": 2048, "numtokens": 256})
CaseKey.default() # default/fallback
"""
def __init__(self, *args: Any, _allow_empty: bool = False, **kwargs: Any):
super().__init__(*args, **kwargs)
if not self and not _allow_empty:
raise TypeError(
"CaseKey requires at least one key-value pair. "
"Use CaseKey.default() for the default config key."
)
self._str: str | None = None
self._hash: int | None = None
@classmethod
def default(cls) -> CaseKey:
"""Create a default case key (empty)."""
return cls(_allow_empty=True)
def __hash__(self) -> int: # type: ignore[override]
if self._hash is None:
self._hash = hash(str(self))
return self._hash
def __str__(self) -> str:
if self._str is None:
self._str = json.dumps(dict(self), sort_keys=True, separators=(",", ":"))
return self._str
def __repr__(self) -> str:
if not self:
return "CaseKey.default()"
return f"CaseKey({dict(self)})"
def is_default(self) -> bool:
"""Return True if this is the default case key (empty)."""
return not self
def _readonly(self, *args: Any, **kwargs: Any) -> Any:
raise TypeError("CaseKey is immutable")
__setitem__ = _readonly # type: ignore[assignment]
__delitem__ = _readonly # type: ignore[assignment]
__ior__ = _readonly # type: ignore[assignment]
update = _readonly # type: ignore[assignment]
pop = _readonly # type: ignore[assignment]
popitem = _readonly # type: ignore[assignment]
setdefault = _readonly # type: ignore[assignment]
clear = _readonly # type: ignore[assignment]
-321
View File
@@ -1,321 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration management for Helion kernels.
This module provides centralized configuration file management for Helion custom
operations, including naming conventions, directory resolution, and file I/O.
Config File Structure
---------------------
Each kernel has a directory: {kernel_name}/
Inside, each GPU platform has its own JSON file: {kernel_name}/{platform}.json
Platform files store config entries as a JSON array::
[
{"key": {}, "config": {...}},
{"key": {"intermediate": 2048, "numtokens": 256}, "config": {...}},
...,
]
Config keys are ``CaseKey`` instances mapping parameter names to
values. The default config uses ``CaseKey.default()``.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"ConfigManager requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
logger = init_logger(__name__)
class ConfigSet:
"""In-memory collection of Helion configs with lookup/query capabilities.
Configs are stored keyed by ``CaseKey``. The default config
uses ``CaseKey.default()`` as its key.
"""
_ConfigDict = dict[str, dict[CaseKey, "helion.Config"]]
def __init__(self, kernel_name: str):
self._kernel_name = kernel_name
self._configs: ConfigSet._ConfigDict = {}
@property
def kernel_name(self) -> str:
return self._kernel_name
def get_config(self, platform: str, config_key: CaseKey) -> helion.Config:
platform_dict = self._configs.get(platform)
if platform_dict is None:
avail_platforms = self.get_platforms()
# TODO(@gmagogsfm): add a CLI/env override flag so users can
# directly specify a platform name instead of relying on
# auto-detection, and suggest it in this error message.
raise KeyError(
f"Config not found for kernel '{self._kernel_name}': "
f"platform '{platform}' not found. "
f"Available platforms: {avail_platforms or '(none)'}. "
f"If your GPU is a variant of a supported platform, "
f"consider adding a mapping in _GPU_NAME_ALIASES in "
f"vllm/kernels/helion/utils.py, or run "
f"tools/autotune_helion_kernels.py to generate configs "
f"for your platform."
)
config = platform_dict.get(config_key)
if config is None:
avail_keys = self.get_config_keys(platform)
raise KeyError(
f"Config not found for kernel '{self._kernel_name}': "
f"config_key '{config_key}' not found for "
f"platform '{platform}'. "
f"Available config_keys: {avail_keys or '(none)'}"
)
return config
def get_platforms(self) -> list[str]:
return sorted(self._configs.keys())
def get_config_keys(self, platform: str) -> list[CaseKey]:
platform_dict = self._configs.get(platform.lower())
if platform_dict is None:
return []
return sorted(platform_dict.keys(), key=str)
def to_config_entries(self) -> dict[str, list[dict[str, Any]]]:
"""Serialize to config entries format for JSON output."""
result: dict[str, list[dict[str, Any]]] = {}
for platform, config_dict in self._configs.items():
pairs: list[dict[str, Any]] = []
for config_key, config in config_dict.items():
config_data = json.loads(config.to_json())
pairs.append({"key": dict(config_key), "config": config_data})
result[platform] = pairs
return result
def to_dict(self) -> dict[str, dict[CaseKey, Any]]:
"""Return configs as a nested dict (platform -> key -> config)."""
result: dict[str, dict[CaseKey, Any]] = {}
for platform, config_dict in self._configs.items():
result[platform] = {
k: json.loads(v.to_json()) for k, v in config_dict.items()
}
return result
@classmethod
def from_dict(cls, kernel_name: str, data: dict[str, Any]) -> ConfigSet:
config_set = cls(kernel_name)
count = 0
for platform, platform_data in data.items():
if platform not in config_set._configs:
config_set._configs[platform] = {}
for entry in platform_data:
raw_key = entry["key"]
key = CaseKey.default() if not raw_key else CaseKey(raw_key)
config = helion.Config(**entry["config"])
config_set._configs[platform][key] = config
count += 1
if count > 0:
logger.debug(
"Loaded %d configs for kernel '%s'",
count,
kernel_name,
)
return config_set
def set_config(
self,
platform: str,
config_key: CaseKey,
config: helion.Config,
) -> None:
platform = platform.lower()
if platform not in self._configs:
self._configs[platform] = {}
self._configs[platform][config_key] = config
logger.debug(
"Set config for kernel '%s': platform='%s', key='%s'",
self._kernel_name,
platform,
config_key,
)
class ConfigManager:
"""File-level configuration management for Helion kernels (global singleton)."""
_instance: ConfigManager | None = None
_instance_base_dir: Path | None = None
def __new__(cls, base_dir: str | Path | None = None) -> ConfigManager:
resolved_base_dir = cls._resolve_base_dir(base_dir)
if cls._instance is not None:
if cls._instance_base_dir != resolved_base_dir:
raise ValueError(
f"ConfigManager singleton already exists with base_dir "
f"'{cls._instance_base_dir}', cannot create with "
f"different base_dir '{resolved_base_dir}'"
)
return cls._instance
instance = super().__new__(cls)
cls._instance = instance
cls._instance_base_dir = resolved_base_dir
return instance
def __init__(self, base_dir: str | Path | None = None):
if hasattr(self, "_base_dir"):
return
self._base_dir = self._resolve_base_dir(base_dir)
logger.debug("ConfigManager initialized with base_dir: %s", self._base_dir)
@staticmethod
def _resolve_base_dir(base_dir: str | Path | None) -> Path:
if base_dir is not None:
return Path(base_dir).resolve()
return (Path(__file__).parent / "configs").resolve()
@classmethod
def get_instance(cls) -> ConfigManager:
if cls._instance is None:
raise RuntimeError(
"ConfigManager instance has not been created. "
"Call ConfigManager(base_dir=...) first to initialize."
)
return cls._instance
@classmethod
def reset_instance(cls) -> None:
"""For testing purposes only."""
cls._instance = None
cls._instance_base_dir = None
def get_kernel_dir(self, kernel_name: str) -> Path:
return self._base_dir / kernel_name
def get_config_file_path(
self, kernel_name: str, platform: str | None = None
) -> Path:
if platform is not None:
return self.get_kernel_dir(kernel_name) / f"{platform}.json"
return self.get_kernel_dir(kernel_name)
def ensure_base_dir_exists(self) -> Path:
self._base_dir.mkdir(parents=True, exist_ok=True)
return self._base_dir
def ensure_base_dir_writable(self) -> None:
self.ensure_base_dir_exists()
test_file = self._base_dir / ".write_test"
try:
test_file.write_text("test")
test_file.unlink()
except OSError as e:
raise OSError(
f"Config directory '{self._base_dir}' is not writable: {e}"
) from e
def _load_platform_file(self, kernel_name: str, platform: str) -> Any:
config_path = self.get_config_file_path(kernel_name, platform)
if not config_path.exists():
return []
try:
with open(config_path) as f:
return json.load(f)
except (json.JSONDecodeError, OSError) as e:
logger.error("Failed to load config file %s: %s", config_path, e)
return []
def load_config_set(self, kernel_name: str) -> ConfigSet:
kernel_dir = self.get_kernel_dir(kernel_name)
if not kernel_dir.is_dir():
return ConfigSet.from_dict(kernel_name, {})
data: dict[str, Any] = {}
for platform_file in sorted(kernel_dir.glob("*.json")):
platform = platform_file.stem
try:
with open(platform_file) as f:
platform_data = json.load(f)
data[platform] = platform_data
except (json.JSONDecodeError, OSError) as e:
logger.error(
"Failed to load config file %s: %s",
platform_file,
e,
)
return ConfigSet.from_dict(kernel_name, data)
def get_platform_configs(
self, kernel_name: str, platform: str
) -> dict[CaseKey, helion.Config]:
platform_data = self._load_platform_file(kernel_name, platform)
if not platform_data:
return {}
config_set = ConfigSet.from_dict(kernel_name, {platform: platform_data})
return {
k: config_set.get_config(platform, k)
for k in config_set.get_config_keys(platform)
}
def save_config_set(self, config_set: ConfigSet) -> Path:
kernel_dir = self.get_kernel_dir(config_set.kernel_name)
kernel_dir.mkdir(parents=True, exist_ok=True)
full_data = config_set.to_config_entries()
for platform, pairs in full_data.items():
platform_path = kernel_dir / f"{platform}.json"
with open(platform_path, "w") as f:
json.dump(pairs, f, indent=2)
f.write("\n")
logger.info("Saved config to: %s", platform_path)
return kernel_dir
def save_configs(
self,
kernel_name: str,
platform: str,
configs: dict[CaseKey, helion.Config],
) -> Path:
"""Save configs for a kernel/platform, merging with existing."""
config_set = ConfigSet.from_dict(
kernel_name,
{platform: self._load_platform_file(kernel_name, platform)},
)
for key, config in configs.items():
config_set.set_config(platform, key, config)
pairs = config_set.to_config_entries().get(platform, [])
platform_path = self.get_config_file_path(kernel_name, platform)
platform_path.parent.mkdir(parents=True, exist_ok=True)
with open(platform_path, "w") as f:
json.dump(pairs, f, indent=2)
f.write("\n")
logger.info("Saved config to: %s", platform_path)
return platform_path
-34
View File
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helion kernel implementation.
Importing this package does NOT register any kernels. Runtime code imports the
specific op module it needs, e.g.::
from vllm.kernels.helion.ops import scaled_mm # noqa: F401
which triggers that op's ``@register_kernel`` as an import side effect.
Tools that need the full registry (e.g. tools/autotune_helion_kernels.py)
call ``import_all_ops()`` to force every op module to register.
"""
import importlib
import pkgutil
def import_all_kernels() -> list[str]:
"""Import every kernel submodule so all ``@register_kernel`` decorators run.
Returns:
The fully-qualified module names that were imported.
"""
imported: list[str] = []
for module_info in pkgutil.iter_modules(__path__):
if module_info.ispkg:
continue
module_name = f"{__name__}.{module_info.name}"
importlib.import_module(module_name)
imported.append(module_name)
return imported
@@ -1,171 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.register import register_kernel
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
logger = init_logger(__name__)
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all input
# property combination. Currently, dtypes are fixed. We need optimization to
# bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
hidden_size_list = [2048, 4096, 5120]
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
inputs = {}
for num_tokens, hidden_size in product(num_tokens_list, hidden_size_list):
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
result = torch.empty(input.shape, device=input.device, dtype=out_dtype)
scale = torch.empty((num_tokens, 1), device=input.device, dtype=scale_dtype)
# scale_ub clamps the per-token amax of |input|. Use a non-degenerate
# upper bound (midway between the mean and max of |input|) so clamping is
# partially active and the baseline comparison is meaningful.
# torch.mean(input) ~= 0 for the zero-mean input would collapse every
# scale to the floor and saturate the output.
input_abs = input.to(torch.float32).abs()
scale_ub = (0.5 * (input_abs.mean() + input_abs.amax())).to(scale_dtype)
config_key = CaseKey({"hidden_size": hidden_size, "num_tokens": num_tokens})
inputs[config_key] = (result, input, scale, scale_ub)
return inputs
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest hidden_size among available configs
(exact match preferred).
2. Among the num_tokens values tuned for that hidden_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
_, input, *_ = args
num_tokens, hidden_size = input.shape
cache_key = (num_tokens, hidden_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, list[int]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["hidden_size"], []).append(key["num_tokens"])
if not configs:
return None
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
available_num_tokens = sorted(configs[best_hidden_size])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey({"hidden_size": best_hidden_size, "num_tokens": best_num_tokens})
_pick_cache[cache_key] = result
return result
def baseline(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
scale: torch.Tensor, # [num_tokens, 1]
scale_ub: torch.Tensor | None = None, # scalar tensor
) -> None:
fp8_min, fp8_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (fp8_max * 512.0)
x = input.to(torch.float32)
s = torch.amax(torch.abs(x), dim=-1, keepdim=True)
if scale_ub is not None:
s = s.clamp(max=scale_ub)
s = (s * (1.0 / fp8_max)).clamp(min=min_scaling_factor)
y = (x / s).clamp(fp8_min, fp8_max)
scale.copy_(s)
result.copy_(y.to(result.dtype))
# Overwrite autotune_baseline_atol and autotune_baseline_rtol
# if too many configs failed due to baseline check during autotuning
@register_kernel(
mutates_args=["result", "scale"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
)
def dynamic_per_token_scaled_fp8_quant(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
scale: torch.Tensor, # [num_tokens, 1]
scale_ub: torch.Tensor | None = None, # scalar tensor
) -> None:
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
num_tokens, hidden_size = input.shape
hl.specialize(hidden_size)
assert result.shape == input.shape
assert scale.shape[0] == num_tokens
assert scale.dtype == torch.float32
assert input.stride()[-1] == 1
assert result.stride()[-1] == 1
fp8_min, fp8_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (fp8_max * 512.0)
for tile_m in hl.tile(num_tokens, block_size=1):
s_blk = hl.zeros([tile_m], dtype=torch.float32)
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(dtype=torch.float32)
tmp_blk = torch.amax(torch.abs(x_blk), dim=-1)
s_blk = torch.maximum(s_blk, tmp_blk)
if scale_ub is not None:
scale_ub_s = hl.load(scale_ub, [])
s_blk = s_blk.clamp(max=scale_ub_s)
s_blk = s_blk * (1.0 / fp8_max)
s_blk = s_blk.clamp(min=min_scaling_factor)
scale[tile_m, 0] = s_blk
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(torch.float32)
y_blk = x_blk * (1.0 / s_blk[:, None])
result[tile_m, tile_n] = y_blk.clamp(fp8_min, fp8_max).to(result.dtype)
@@ -1,297 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm import ir
from vllm.kernels.helion.register import register_kernel
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
logger = init_logger(__name__)
def _compute_cos_sin_cache(
max_position_embeddings, rotary_dim, device="cuda", dtype=torch.float
):
inv_freq = 1.0 / (
10000
** (torch.arange(0, rotary_dim, 2, device=device, dtype=dtype) / rotary_dim)
)
t = torch.arange(max_position_embeddings, device=device, dtype=dtype)
freqs = torch.einsum("i,j -> ij", t, inv_freq)
cos = freqs.cos()
sin = freqs.sin()
cache = torch.cat((cos, sin), dim=-1)
return cache
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover
# all input property combination. Currently, dtypes are fixed. We need
# optimization to bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
num_heads_pair = [
(16, 8),
(32, 8),
(64, 8),
]
head_dim = 128
in_dtype: torch.dtype = torch.bfloat16
rotary_ratio = 1.0
is_neox = True
eps = 1e-6
device = "cuda"
inputs = {}
for num_tokens, (num_q_heads, num_kv_heads) in product(
num_tokens_list, num_heads_pair
):
total_dim = (num_q_heads + 2 * num_kv_heads) * head_dim
qkv = torch.empty(
num_tokens, total_dim, dtype=in_dtype, device=device
).uniform_(-0.1, 0.1)
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
q_weight = torch.empty(head_dim, dtype=in_dtype, device=device).uniform_(
0.8, 1.2
)
k_weight = torch.empty(head_dim, dtype=in_dtype, device=device).uniform_(
0.8, 1.2
)
rotary_dim = int(head_dim * rotary_ratio)
cos_sin_cache = _compute_cos_sin_cache(40960, rotary_dim)
cos_sin_cache = cos_sin_cache.to(in_dtype)
config_key = CaseKey(
{
"q_heads": num_q_heads,
"kv_heads": num_kv_heads,
"num_tokens": num_tokens,
}
)
inputs[config_key] = (
qkv,
num_q_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
cos_sin_cache,
is_neox,
positions.view(-1),
)
return inputs
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest q_heads among available configs
(exact match preferred).
2. Find the closest kv_heads among available configs
(exact match preferred).
3. Among the num_tokens values tuned for that q_heads and q_heads, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
qkv, q_heads, kv_heads, *_ = args
num_tokens = qkv.shape[0]
cache_key = (num_tokens, q_heads, kv_heads)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, dict[int, list[int]]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["q_heads"], {}).setdefault(key["kv_heads"], []).append(
key["num_tokens"]
)
if not configs:
return None
best_q_heads = min(configs, key=lambda s: abs(s - q_heads))
best_kv_heads = min(configs[best_q_heads], key=lambda s: abs(s - kv_heads))
available_num_tokens = sorted(configs[best_q_heads][best_kv_heads])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey(
{
"q_heads": best_q_heads,
"kv_heads": best_kv_heads,
"num_tokens": best_num_tokens,
}
)
_pick_cache[cache_key] = result
return result
def baseline(
qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
num_heads_q: int,
num_heads_k: int,
num_heads_v: int,
head_dim: int,
eps: float,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor, # [max_position, rotary_dim]
is_neox: bool,
position_ids: torch.Tensor, # [num_tokens],
forced_token_heads_per_warp: int = -1, # dummy
) -> None:
q_size = num_heads_q * head_dim
kv_size = num_heads_k * head_dim
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
q_by_head = q.view(*q.shape[:-1], q.shape[-1] // head_dim, head_dim)
q_by_head = ir.ops.rms_norm(q_by_head, q_weight, eps)
q = q_by_head.view(q.shape)
k_by_head = k.view(*k.shape[:-1], k.shape[-1] // head_dim, head_dim)
k_by_head = ir.ops.rms_norm(k_by_head, k_weight, eps)
k = k_by_head.view(k.shape)
q, k = RotaryEmbedding.forward_static(
position_ids, q, k, head_dim, cos_sin_cache.shape[1], cos_sin_cache, is_neox
)
qkv[:, :q_size].copy_(q)
qkv[:, q_size : q_size + kv_size].copy_(k)
# Overwrite autotune_baseline_atol and autotune_baseline_rtol
# if too many configs failed due to baseline check during autotuning
@register_kernel(
mutates_args=["qkv"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
autotune_baseline_atol=5e-2,
autotune_baseline_rtol=5e-2,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
) # type: ignore[misc]
def fused_qk_norm_rope(
qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
num_heads_q: int,
num_heads_k: int,
num_heads_v: int,
head_dim: int,
eps: float,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor, # [max_position, rotary_dim]
is_neox: bool,
position_ids: torch.Tensor, # [num_tokens],
forced_token_heads_per_warp: int = -1, # dummy
) -> None:
assert qkv.ndim == 2
num_tokens = qkv.shape[0]
total_heads = num_heads_q + num_heads_k + num_heads_v
assert qkv.shape[1] == total_heads * head_dim
hl.specialize(qkv.shape[1])
assert cos_sin_cache.ndim == 2
max_position, rotary_dim = cos_sin_cache.shape
hl.specialize(max_position)
hl.specialize(rotary_dim)
assert rotary_dim % 2 == 0
assert rotary_dim <= head_dim
embed_dim = rotary_dim // 2
hl.specialize(num_heads_q)
hl.specialize(num_heads_k)
hl.specialize(num_heads_v)
hl.specialize(head_dim)
assert position_ids.ndim == 1 and position_ids.shape[0] == num_tokens
hl.specialize(position_ids.shape[0])
assert q_weight.ndim == 1 and q_weight.shape[0] == head_dim
hl.specialize(q_weight.shape[0])
assert k_weight.ndim == 1 and k_weight.shape[0] == head_dim
hl.specialize(k_weight.shape[0])
assert qkv.dtype == q_weight.dtype and q_weight.dtype == k_weight.dtype
assert position_ids.dtype == torch.int64
assert qkv.is_contiguous()
assert position_ids.is_contiguous()
assert q_weight.is_contiguous()
assert k_weight.is_contiguous()
assert cos_sin_cache.is_contiguous()
qk_heads = num_heads_q + num_heads_k
qkv = qkv.view(num_tokens, -1, head_dim)
for tile_m, tile_gn, tile_n in hl.tile(
[num_tokens, qk_heads, head_dim], block_size=[1, None, head_dim]
):
x_blk = qkv[tile_m, tile_gn, tile_n].to(dtype=torch.float32)
rms = x_blk.pow(2).sum(dim=-1)
rms = torch.rsqrt(rms * (1.0 / head_dim) + eps)
use_q_weight = (tile_gn.index < num_heads_q)[None, :, None]
w_blk = torch.where(
use_q_weight, q_weight[None, None, tile_n], k_weight[None, None, tile_n]
)
x_blk = (x_blk * rms[:, :, None]).to(qkv.dtype) * w_blk
qkv[tile_m, tile_gn, tile_n] = x_blk
pos_id = position_ids[tile_m]
cos_blk = cos_sin_cache[pos_id, hl.arange(embed_dim)]
sin_blk = cos_sin_cache[pos_id, hl.arange(embed_dim) + embed_dim]
if is_neox:
x1_offset = hl.arange(embed_dim)
x2_offset = x1_offset + embed_dim
else:
x1_offset = hl.arange(embed_dim) * 2
x2_offset = x1_offset + 1
x1_blk = qkv[tile_m, tile_gn, x1_offset]
x2_blk = qkv[tile_m, tile_gn, x2_offset]
o1_blk = x1_blk * cos_blk[:, None, :] - x2_blk * sin_blk[:, None, :]
o2_blk = x2_blk * cos_blk[:, None, :] + x1_blk * sin_blk[:, None, :]
qkv[tile_m, tile_gn, x1_offset] = o1_blk
qkv[tile_m, tile_gn, x2_offset] = o2_blk
@@ -1,212 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all
# input property combination. Currently, dtypes are fixed. We need
# optimization to bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
hidden_size_list = [2048, 4096, 5120]
group_size_list = [128]
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
use_ue8m0 = False
column_major = False
fp8_min, fp8_max = get_fp8_min_max()
eps = 1e-10
inputs = {}
for hidden_size, group_size, num_tokens in product(
hidden_size_list, group_size_list, num_tokens_list
):
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
output_q = torch.empty(input.shape, device=input.device, dtype=out_dtype)
output_s = torch.empty(
(num_tokens, hidden_size // group_size),
device=input.device,
dtype=scale_dtype,
)
config_key = CaseKey(
{
"hidden_size": hidden_size,
"group_size": group_size,
"num_tokens": num_tokens,
}
)
inputs[config_key] = (
input,
output_q,
output_s,
group_size,
eps,
fp8_min,
fp8_max,
use_ue8m0,
column_major,
False,
)
return inputs
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest hidden_size among available configs
(exact match preferred).
2. Find the closest group_size among available configs
(exact match preferred).
3. Among the num_tokens values tuned for that hidden_size and group_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
input, _, _, group_size, *_ = args
num_tokens, hidden_size = input.shape
cache_key = (num_tokens, group_size, hidden_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, dict[int, list[int]]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["hidden_size"], {}).setdefault(
key["group_size"], []
).append(key["num_tokens"])
if not configs:
return None
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
best_group_size = min(configs[best_hidden_size], key=lambda s: abs(s - group_size))
available_num_tokens = sorted(configs[best_hidden_size][best_group_size])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey(
{
"hidden_size": best_hidden_size,
"group_size": best_group_size,
"num_tokens": best_num_tokens,
}
)
_pick_cache[cache_key] = result
return result
def baseline(
input: torch.Tensor, # [num_tokens, hidden_size]
output_q: torch.Tensor, # [num_tokens, hidden_size]
output_s: torch.Tensor, # [num_tokens, groups_per_row]
group_size: int,
eps: float,
fp8_min: float,
fp8_max: float,
scale_ue8m0: bool,
dummy_is_scale_transposed: bool = False,
dummy_is_tma_aligned: bool = False,
) -> None:
num_tokens, hidden_size = input.shape
groups_per_row = hidden_size // group_size
x = input.view(num_tokens, groups_per_row, group_size).to(torch.float32)
s = torch.clamp(torch.amax(torch.abs(x), dim=-1), min=eps) / fp8_max
if scale_ue8m0:
s = torch.exp2(torch.ceil(torch.log2(s)))
y = torch.clamp(x / s[:, :, None], fp8_min, fp8_max)
output_s.copy_(s)
output_q.copy_(y.view(num_tokens, hidden_size).to(output_q.dtype))
@register_kernel(
mutates_args=["output_q", "output_s"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
),
) # type: ignore[misc]
def per_token_group_fp8_quant(
input: torch.Tensor, # [num_tokens, hidden_size]
output_q: torch.Tensor, # [num_tokens, hidden_size]
output_s: torch.Tensor, # [num_tokens, groups_per_row]
group_size: int,
eps: float,
fp8_min: float,
fp8_max: float,
scale_ue8m0: bool,
# Unused dummy args
# Kept for consistency with existing kernel interface
dummy_is_scale_transposed: bool = False,
dummy_is_tma_aligned: bool = False,
) -> None:
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
num_tokens, hidden_size = input.shape
hl.specialize(hidden_size)
hl.specialize(group_size)
groups_per_row = output_s.shape[1]
hl.specialize(groups_per_row)
assert hidden_size % group_size == 0 and hidden_size // group_size == groups_per_row
assert output_s.ndim == 2 and output_s.dtype == torch.float32
input = input.view(num_tokens, -1, group_size)
output_q = output_q.view(num_tokens, -1, group_size)
for tile_m, tile_gn, tile_n in hl.tile(
[num_tokens, groups_per_row, group_size], block_size=[1, None, group_size]
):
x_blk = input[tile_m, tile_gn, tile_n]
y_s_blk = torch.clamp(torch.amax(torch.abs(x_blk), dim=-1), min=eps)
y_s_blk = y_s_blk / fp8_max
if scale_ue8m0:
y_s_blk = torch.exp2(torch.ceil(torch.log2(y_s_blk)))
y_q_blk = torch.clamp(x_blk / y_s_blk[:, :, None], fp8_min, fp8_max).to(
output_q.dtype
)
output_s[tile_m, tile_gn] = y_s_blk
output_q[tile_m, tile_gn, tile_n] = y_q_blk
@@ -1,256 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.utils import (
get_fp8_dtype,
get_int8_min_max,
get_int8_min_scaling_factor,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all
# input property combination. Currently, dtypes are fixed. We need
# optimization to bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
hidden_size_list = [2048, 4096, 5120]
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
inputs = {}
for num_tokens, hidden_size in product(num_tokens_list, hidden_size_list):
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
result = torch.empty(input.shape, device=input.device, dtype=out_dtype)
scale = torch.empty((num_tokens, 1), device=input.device, dtype=scale_dtype)
residual = torch.randn_like(input)
weight = torch.normal(
mean=1.0,
std=1.0,
size=(hidden_size,),
dtype=input.dtype,
device=input.device,
)
epsilon = 1e-6
# scale_ub clamps the per-token amax of the RMS-normed, weighted output.
# Use a non-degenerate upper bound (midway between the mean and max of
# that magnitude) so clamping is partially active and the baseline
# comparison is meaningful. torch.mean(input) ~= 0 for the zero-mean
# input would collapse every scale to the floor and saturate the output.
# Mirrors the reference normalization in baseline() below.
x = input.to(torch.float32) + residual.to(torch.float32)
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
x_norm_abs = ((x * rms).to(input.dtype) * weight).abs().to(torch.float32)
scale_ub = (0.5 * (x_norm_abs.mean() + x_norm_abs.amax())).to(scale_dtype)
config_key = CaseKey({"hidden_size": hidden_size, "num_tokens": num_tokens})
inputs[config_key] = (result, input, weight, scale, epsilon, scale_ub, residual)
return inputs
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest hidden_size among available configs
(exact match preferred).
2. Among the num_tokens values tuned for that hidden_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
_, input, *_ = args
num_tokens, hidden_size = input.shape
cache_key = (num_tokens, hidden_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, list[int]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["hidden_size"], []).append(key["num_tokens"])
if not configs:
return None
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
available_num_tokens = sorted(configs[best_hidden_size])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey({"hidden_size": best_hidden_size, "num_tokens": best_num_tokens})
_pick_cache[cache_key] = result
return result
def baseline(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
weight: torch.Tensor, # [num_tokens]
scale: torch.Tensor, # [num_tokens, 1]
epsilon: float,
scale_ub: torch.Tensor | None = None, # []
residual: torch.Tensor | None = None, # [num_tokens, hidden_size]
) -> None:
_, hidden_size = input.shape
quant_dtype = result.dtype
qtype_min: int | float
qtype_max: int | float
if quant_dtype == torch.int8:
qtype_min, qtype_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_min, qtype_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_max * 512.0)
x = input.to(torch.float32)
if residual is not None:
x = x + residual
residual.copy_(x.to(residual.dtype))
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
x_norm = (x * rms).to(input.dtype) * weight
s = torch.amax(torch.abs(x_norm), dim=-1, keepdim=True).to(torch.float32)
if scale_ub is not None:
s = s.clamp(max=scale_ub)
s = (s * (1.0 / qtype_max)).clamp(min=min_scaling_factor)
y = x_norm / s
if quant_dtype == torch.int8:
y = y.round()
scale.copy_(s)
result.copy_(y.clamp(qtype_min, qtype_max).to(result.dtype))
# Overwrite autotune_baseline_atol and autotune_baseline_rtol
# if too many configs failed due to baseline check during autotuning
@register_kernel(
mutates_args=["result", "scale", "residual"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
) # type: ignore[misc]
def rms_norm_dynamic_per_token_quant(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
weight: torch.Tensor, # [hidden_size]
scale: torch.Tensor, # [num_tokens, 1]
epsilon: float,
scale_ub: torch.Tensor | None = None, # []
residual: torch.Tensor | None = None, # [num_tokens, hidden_size]
) -> None:
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
num_tokens, hidden_size = input.shape
hl.specialize(hidden_size)
fp8_dtype = get_fp8_dtype()
assert result.dtype in [fp8_dtype, torch.int8]
assert result.is_contiguous() and input.is_contiguous()
if scale_ub is not None:
assert result.dtype == fp8_dtype
assert scale_ub.dtype == torch.float32
assert input.dtype == weight.dtype
assert scale.shape[0] == num_tokens
assert scale.dtype == torch.float32
if residual is not None:
assert residual.dtype == input.dtype
quant_dtype = result.dtype
qtype_traits_min: int | float
qtype_traits_max: int | float
if quant_dtype == torch.int8:
qtype_traits_min, qtype_traits_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_traits_max * 512.0)
qtype_max = float(qtype_traits_max)
for tile_m in hl.tile(num_tokens, block_size=1):
rms = hl.zeros([tile_m], dtype=torch.float32)
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(torch.float32)
if residual is not None:
x_blk = x_blk + residual[tile_m, tile_n]
rms = rms + x_blk.pow(2).sum(dim=-1)
rms = torch.rsqrt(rms * (1.0 / hidden_size) + epsilon)
s_blk = hl.zeros([tile_m], dtype=torch.float32)
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(torch.float32)
if residual is not None:
x_blk = x_blk + residual[tile_m, tile_n]
x_blk = (x_blk * rms[:, None]).to(input.dtype) * weight[None, tile_n]
tmp_blk = torch.amax(torch.abs(x_blk), dim=-1).to(torch.float32)
s_blk = torch.maximum(s_blk, tmp_blk)
if scale_ub is not None:
scale_ub_s = hl.load(scale_ub, [])
s_blk = s_blk.clamp(max=scale_ub_s)
s_blk = s_blk * (1.0 / qtype_max)
s_blk = s_blk.clamp(min=min_scaling_factor)
scale[tile_m, 0] = s_blk
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(torch.float32)
if residual is not None:
x_blk = x_blk + residual[tile_m, tile_n]
residual[tile_m, tile_n] = x_blk.to(residual.dtype)
x_blk = (x_blk * rms[:, None]).to(input.dtype) * weight[None, tile_n]
if quant_dtype == torch.int8:
s_inv_blk = 1.0 / s_blk[:, None]
y_blk = x_blk * s_inv_blk
y_blk = y_blk.round()
else:
y_blk = x_blk / s_blk[:, None]
result[tile_m, tile_n] = y_blk.clamp(qtype_traits_min, qtype_traits_max).to(
result.dtype
)
@@ -1,316 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.utils import (
get_fp8_dtype,
get_int8_min_max,
get_int8_min_scaling_factor,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all
# input property combination. Currently, dtypes are fixed. We need
# optimization to bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
hidden_size_list = [2048, 4096, 5120]
group_size_list = [128]
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
inputs = {}
for hidden_size, group_size, num_tokens in product(
hidden_size_list, group_size_list, num_tokens_list
):
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
result = torch.empty(input.shape, device=input.device, dtype=out_dtype)
scale = torch.empty(
(num_tokens, hidden_size // group_size),
device=input.device,
dtype=scale_dtype,
)
residual = torch.randn_like(input)
weight = torch.normal(
mean=1.0,
std=1.0,
size=(hidden_size,),
dtype=input.dtype,
device=input.device,
)
epsilon = 1e-6
# scale_ub clamps the per-group amax of the RMS-normed, weighted output.
# Use a non-degenerate upper bound (midway between the mean and max of
# that magnitude) so clamping is partially active and the baseline
# comparison is meaningful. torch.mean(input) ~= 0 for the zero-mean
# input would collapse every scale to the floor and saturate the output.
# Mirrors the reference normalization in baseline() below.
x = input.to(torch.float32) + residual.to(torch.float32)
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
x_norm_abs = ((x * rms).to(input.dtype) * weight).abs().to(torch.float32)
scale_ub = (0.5 * (x_norm_abs.mean() + x_norm_abs.amax())).to(scale_dtype)
config_key = CaseKey(
{
"hidden_size": hidden_size,
"group_size": group_size,
"num_tokens": num_tokens,
}
)
inputs[config_key] = (
result,
input,
weight,
scale,
epsilon,
scale_ub,
residual,
group_size,
False,
)
return inputs
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest hidden_size among available configs
(exact match preferred).
2. Find the closest group_size among available configs
(exact match preferred).
2. Among the num_tokens values tuned for that hidden_size and group_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
_, input, _, _, _, _, _, group_size, *_ = args
num_tokens, hidden_size = input.shape
cache_key = (num_tokens, group_size, hidden_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, dict[int, list[int]]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["hidden_size"], {}).setdefault(
key["group_size"], []
).append(key["num_tokens"])
if not configs:
return None
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
best_group_size = min(configs[best_hidden_size], key=lambda s: abs(s - group_size))
available_num_tokens = sorted(configs[best_hidden_size][best_group_size])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey(
{
"hidden_size": best_hidden_size,
"group_size": best_group_size,
"num_tokens": best_num_tokens,
}
)
_pick_cache[cache_key] = result
return result
def baseline(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
weight: torch.Tensor, # [hidden_size]
scale: torch.Tensor, # [num_tokens, groups_per_row]
epsilon: float,
scale_ub: torch.Tensor | None, # []
residual: torch.Tensor | None, # [num_tokens, hidden_size]
group_size: int,
is_scale_transposed: bool,
) -> None:
num_tokens, hidden_size = input.shape
groups_per_row = hidden_size // group_size
quant_dtype = result.dtype
qtype_min: int | float
qtype_max: int | float
if quant_dtype == torch.int8:
qtype_min, qtype_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_min, qtype_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_max * 512.0)
x = input.to(torch.float32)
if residual is not None:
x = x + residual
residual.copy_(x.to(residual.dtype))
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
x_norm = (x * rms).to(input.dtype) * weight
x_grouped = x_norm.view(num_tokens, groups_per_row, group_size).to(torch.float32)
s = torch.amax(torch.abs(x_grouped), dim=-1).to(torch.float32)
if scale_ub is not None:
s = s.clamp(max=scale_ub)
s = (s * (1.0 / qtype_max)).clamp(min=min_scaling_factor)
y = x_grouped / s[:, :, None]
if quant_dtype == torch.int8:
y = y.round()
scale.copy_(s)
result.copy_(
y.clamp(qtype_min, qtype_max).view(num_tokens, hidden_size).to(result.dtype)
)
@register_kernel(
mutates_args=["result", "scale", "residual"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
) # type: ignore[misc]
def rms_norm_per_block_quant(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
weight: torch.Tensor, # [hidden_size]
scale: torch.Tensor, # [num_tokens, groups_per_row]
epsilon: float,
scale_ub: torch.Tensor | None, # []
residual: torch.Tensor | None, # [num_tokens, hidden_size]
group_size: int,
is_scale_transposed: bool, # dummy
) -> None:
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
num_tokens, hidden_size = input.shape
hl.specialize(hidden_size)
hl.specialize(group_size)
groups_per_row = scale.shape[1]
hl.specialize(groups_per_row)
assert hidden_size % group_size == 0 and hidden_size // group_size == groups_per_row
assert scale.shape[0] == num_tokens
assert scale.dtype == torch.float32
if scale.stride(1) > 1:
assert is_scale_transposed
fp8_dtype = get_fp8_dtype()
assert result.dtype in [fp8_dtype, torch.int8]
assert result.is_contiguous() and input.is_contiguous()
if scale_ub is not None:
assert result.dtype == fp8_dtype
assert scale_ub.dtype == torch.float32
assert input.dtype == weight.dtype
if residual is not None:
assert residual.dtype == input.dtype
assert group_size in [64, 128]
quant_dtype = result.dtype
qtype_traits_min: int | float
qtype_traits_max: int | float
if quant_dtype == torch.int8:
qtype_traits_min, qtype_traits_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_traits_max * 512.0)
qtype_max = float(qtype_traits_max)
for tile_m in hl.tile(num_tokens, block_size=1):
rms = hl.zeros([tile_m], dtype=torch.float32)
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(torch.float32)
if residual is not None:
x_blk = x_blk + residual[tile_m, tile_n]
rms = rms + x_blk.pow(2).sum(dim=-1)
rms = torch.rsqrt(rms * (1.0 / hidden_size) + epsilon)
m_idx = tile_m.begin + hl.arange(tile_m.block_size)
m_blk = m_idx[:, None, None]
for tile_gn, tile_n in hl.tile(
[groups_per_row, group_size], block_size=[None, group_size]
):
gn_idx = tile_gn.index
n_offset = tile_n.index
n_idx = gn_idx[:, None] * group_size + n_offset[None, :]
n_blk = n_idx[None, :, :]
mask = (gn_idx < groups_per_row)[None, :, None]
x_blk = hl.load(input, [m_blk, n_blk], extra_mask=mask).to(
dtype=torch.float32
)
if residual is not None:
r_blk = hl.load(residual, [m_blk, n_blk], extra_mask=mask)
x_blk = x_blk + r_blk
w_blk = hl.load(weight, [n_blk], extra_mask=mask)
x_norm_blk = (x_blk * rms[:, None, None]).to(input.dtype) * w_blk
s_blk = torch.amax(torch.abs(x_norm_blk), dim=-1).to(torch.float32)
if scale_ub is not None:
scale_ub_s = hl.load(scale_ub, [])
s_blk = s_blk.clamp(max=scale_ub_s)
s_blk = s_blk * (1.0 / qtype_max)
s_blk = s_blk.clamp(min=min_scaling_factor)
scale[tile_m, tile_gn] = s_blk
if quant_dtype == torch.int8:
y_blk = (x_norm_blk * (1.0 / s_blk[:, :, None])).round()
else:
y_blk = x_norm_blk / s_blk[:, :, None]
y_blk = y_blk.clamp(qtype_traits_min, qtype_traits_max).to(result.dtype)
hl.store(result, [m_blk, n_blk], y_blk, extra_mask=mask)
if residual is not None:
hl.store(
residual, [m_blk, n_blk], x_blk.to(residual.dtype), extra_mask=mask
)
@@ -1,273 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.utils import (
get_fp8_dtype,
get_int8_min_max,
get_int8_min_scaling_factor,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all input
# property combination. Currently, dtypes are fixed. We need optimization to
# bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
intermediate_size_list = [6144, 12288, 25600]
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
group_size_list = [128]
inputs = {}
for intermediate_size, group_size, num_tokens in product(
intermediate_size_list, group_size_list, num_tokens_list
):
input = torch.randn(
num_tokens, 2 * intermediate_size, device="cuda", dtype=in_dtype
)
result = torch.empty(
num_tokens, intermediate_size, device=input.device, dtype=out_dtype
)
scale = torch.empty(
(num_tokens, intermediate_size // group_size),
device=input.device,
dtype=scale_dtype,
)
# scale_ub clamps the per-group amax of the SiLU-and-mul activation. Use
# a non-degenerate upper bound (midway between the mean and max of the
# activation magnitude) so clamping is partially active and the baseline
# comparison is meaningful. torch.mean(input) ~= 0 for the zero-mean
# input would collapse every scale to the floor and saturate the output.
# Mirrors tests/kernels/helion/test_silu_and_mul_per_block_quant.py.
act_abs = SiluAndMul.forward_native(input.to(torch.float32)).abs()
scale_ub = (0.5 * (act_abs.mean() + act_abs.amax())).to(scale_dtype)
config_key = CaseKey(
{
"intermediate_size": intermediate_size,
"group_size": group_size,
"num_tokens": num_tokens,
}
)
inputs[config_key] = (result, input, scale, group_size, scale_ub, False)
return inputs
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest intermediate_size among available configs
(exact match preferred).
2. Find the closest group_size among available configs
(exact match preferred).
3. Among the num_tokens values tuned for that intermediate_size and group_size,
pick the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
result, _, _, group_size, *_ = args
num_tokens, intermediate_size = result.shape
cache_key = (num_tokens, group_size, intermediate_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, dict[int, list[int]]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["intermediate_size"], {}).setdefault(
key["group_size"], []
).append(key["num_tokens"])
if not configs:
return None
best_intermediate_size = min(configs, key=lambda s: abs(s - intermediate_size))
best_group_size = min(
configs[best_intermediate_size], key=lambda s: abs(s - group_size)
)
available_num_tokens = sorted(configs[best_intermediate_size][best_group_size])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey(
{
"intermediate_size": best_intermediate_size,
"group_size": best_group_size,
"num_tokens": best_num_tokens,
}
)
_pick_cache[cache_key] = result
return result
def baseline(
out: torch.Tensor, # [num_tokens, intermediate_size]
input: torch.Tensor, # [num_tokens, 2 * intermediate_size]
scales: torch.Tensor, # [num_tokens, groups_per_row]
group_size: int,
scale_ub: torch.Tensor | None = None, # scalar tensor
is_scale_transposed: bool = False,
) -> None:
num_tokens, intermediate_size = out.shape
groups_per_row = intermediate_size // group_size
quant_dtype = out.dtype
qtype_min: int | float
qtype_max: int | float
if quant_dtype == torch.int8:
qtype_min, qtype_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_min, qtype_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_max * 512.0)
act = SiluAndMul.forward_native(input.to(torch.float32))
x_grouped = act.view(num_tokens, groups_per_row, group_size)
s = torch.amax(torch.abs(x_grouped), dim=-1)
if scale_ub is not None:
s = s.clamp(max=scale_ub)
s = (s * (1.0 / qtype_max)).clamp(min=min_scaling_factor)
y = x_grouped / s[:, :, None]
if quant_dtype == torch.int8:
y = y.round()
scales.copy_(s)
out.copy_(
y.clamp(qtype_min, qtype_max).view(num_tokens, intermediate_size).to(out.dtype)
)
@register_kernel(
mutates_args=["out", "scales"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
) # type: ignore[misc]
def silu_and_mul_per_block_quant(
out: torch.Tensor, # [num_tokens, intermediate_size]
input: torch.Tensor, # [num_tokens, 2 * intermediate_size]
scales: torch.Tensor, # [num_tokens, groups_per_row]
group_size: int,
scale_ub: torch.Tensor | None = None, # scalar tensor
is_scale_transposed: bool = False, # dummy
) -> None:
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
num_tokens, two_intermediate_size = input.shape
hl.specialize(two_intermediate_size)
assert two_intermediate_size % 2 == 0
intermediate_size = two_intermediate_size // 2
assert out.shape[0] == num_tokens
assert out.shape[1] == intermediate_size
fp8_dtype = get_fp8_dtype()
assert out.dtype in [fp8_dtype, torch.int8]
if scale_ub is not None:
assert out.dtype == fp8_dtype
assert scale_ub.dtype == torch.float32
assert scales.ndim == 2 and scales.dtype == torch.float32
assert scales.shape[0] == num_tokens
groups_per_row = scales.shape[1]
hl.specialize(groups_per_row)
assert (
intermediate_size % group_size == 0
and intermediate_size // group_size == groups_per_row
)
assert group_size in [64, 128]
hl.specialize(group_size)
assert input.stride()[-1] == 1
assert out.stride()[-1] == 1
quant_dtype = out.dtype
qtype_traits_min: int | float
qtype_traits_max: int | float
if quant_dtype == torch.int8:
qtype_traits_min, qtype_traits_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_traits_max * 512.0)
qtype_max = float(qtype_traits_max)
input = input.view(num_tokens, -1, group_size)
out = out.view(num_tokens, -1, group_size)
for tile_m, tile_gn, tile_n in hl.tile(
[num_tokens, groups_per_row, group_size], block_size=[1, None, group_size]
):
x_a_blk = input[tile_m, tile_gn, tile_n].to(torch.float32)
x_b_blk = hl.load(
input,
[tile_m, tile_gn.index + groups_per_row, tile_n],
extra_mask=(tile_gn.index + groups_per_row < 2 * groups_per_row)[
None, :, None
],
).to(torch.float32)
x_blk = x_a_blk * torch.sigmoid(x_a_blk) * x_b_blk
s_blk = torch.amax(torch.abs(x_blk), dim=-1).to(torch.float32)
if scale_ub is not None:
scale_ub_s = hl.load(scale_ub, [])
s_blk = s_blk.clamp(max=scale_ub_s)
s_blk = s_blk * (1.0 / qtype_max)
s_blk = s_blk.clamp(min=min_scaling_factor)
scales[tile_m, tile_gn] = s_blk
if quant_dtype == torch.int8:
y_blk = (x_blk * (1.0 / s_blk[:, :, None])).round()
else:
y_blk = x_blk / s_blk[:, :, None]
out[tile_m, tile_gn, tile_n] = y_blk.clamp(
qtype_traits_min, qtype_traits_max
).to(out.dtype)
-135
View File
@@ -1,135 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"silu_mul_fp8 Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion.language as hl
from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_silu_mul_fp8_inputs() -> dict[CaseKey, tuple[Any, ...]]:
intermediate_sizes = [2048, 2880, 4096, 8192, 11008, 14336]
# Use the same num_tokens values as vLLM's default cudagraph capture sizes.
# See vllm/config/vllm.py _set_cudagraph_sizes() for the canonical formula.
num_tokens_list = [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, 513, 16))
inputs: dict[CaseKey, tuple[Any, ...]] = {}
for num_tokens in num_tokens_list:
for intermediate_size in intermediate_sizes:
input_tensor = torch.randn(
num_tokens,
2 * intermediate_size,
device="cuda",
dtype=torch.bfloat16,
)
scale = torch.tensor([1.0], device="cuda", dtype=torch.float32)
key = CaseKey({"intermediate": intermediate_size, "numtokens": num_tokens})
inputs[key] = (input_tensor, scale)
return inputs
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
def pick_silu_mul_fp8_config(
args: tuple[Any, ...], config_keys: list[CaseKey]
) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest intermediate_size among available configs
(exact match preferred).
2. Among the num_tokens values tuned for that intermediate_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
input_tensor, _scale = args
intermediate_size = int(input_tensor.shape[-1]) // 2
num_tokens = int(input_tensor.view(-1, input_tensor.shape[-1]).shape[0])
cache_key = (num_tokens, intermediate_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
by_isize: dict[int, list[int]] = {}
for k in config_keys:
if k.is_default():
continue
by_isize.setdefault(k["intermediate"], []).append(k["numtokens"])
if not by_isize:
return None
best_isize = min(by_isize, key=lambda s: abs(s - intermediate_size))
available = sorted(by_isize[best_isize])
best_ntokens = next((n for n in available if n >= num_tokens), available[-1])
result = CaseKey({"intermediate": best_isize, "numtokens": best_ntokens})
_pick_cache[cache_key] = result
return result
@register_kernel(
config_picker=pick_silu_mul_fp8_config,
input_generator=generate_silu_mul_fp8_inputs,
)
def silu_mul_fp8(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
original_shape = input.shape
two_d = hl.specialize(original_shape[-1])
d = two_d // 2
output_shape = original_shape[:-1] + (d,)
input_2d = input.view(-1, original_shape[-1])
m = input_2d.shape[0]
# TODO(gmagogsfm): Support for more float8 subtypes (e4m3fnuz, e5m2) coming
out = torch.empty((m, d), device=input.device, dtype=torch.float8_e4m3fn)
input_part_a = input_2d[:, :d]
input_part_b = input_2d[:, d:]
assert scale.numel() == 1, "Scale must be a scalar Tensor"
for tile_m, tile_n in hl.tile([m, d]):
a_vals = input_part_a[tile_m, tile_n]
silu_result = torch.nn.functional.silu(a_vals)
b_vals = input_part_b[tile_m, tile_n]
result = silu_result * b_vals
result_f32 = result.to(torch.float32)
scale_val = hl.load(scale, [0])
inv_scale = 1.0 / scale_val
result_scaled = result_f32 * inv_scale
out[tile_m, tile_n] = result_scaled.to(out.dtype)
return out.view(output_shape)
def silu_mul_fp8_baseline(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
output_shape = input.shape[:-1] + (input.shape[-1] // 2,)
out = torch.empty(output_shape, dtype=torch.float8_e4m3fn, device=input.device)
torch.ops._C.silu_and_mul_quant(out, input, scale)
return out
-513
View File
@@ -1,513 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM Helion kernel registration with pre-tuned config selection.
This module leverages Helion's internal config selection infrastructure to use
pre-tuned configs instead of runtime autotuning.
How Helion Normally Works
-------------------------
For each kernel invocation, Helion:
1. Computes a cache key from input arguments
2. Looks up the key in its internal compilation cache
3. On cache miss, runs autotuning to find the best config
4. Compiles and caches the kernel with that config
How We Override It
------------------
We override two Helion hooks to use pre-tuned configs:
1. **key**: We provide a key function (derived from config_picker) that
computes cache keys matching our pre-tuned config keys. This ensures Helion's
internal cache uses keys that correspond to configs we've prepared.
2. **autotuner_fn**: We provide PresetConfigSearch which, instead of autotuning,
simply returns the pre-tuned config for the computed key. On cache miss,
Helion calls our autotuner which returns the author-prepared config.
Both hooks use the same config_picker logic to ensure the cache key computed
by key matches the config returned by the autotuner.
Key Classes
-----------
- HelionKernelWrapper: Wraps raw kernel + config_picker, creates configured kernels
- ConfiguredHelionKernel: Platform-specific kernel with pre-tuned configs
- PresetConfigSearch: Custom autotuner that returns pre-tuned configs
"""
from __future__ import annotations
import inspect
from collections.abc import Callable
from typing import Any
import torch
from torch.library import Library
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
from vllm.utils.torch_utils import direct_register_custom_op
if not has_helion():
raise ImportError(
"register module requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
from helion.autotuner.base_search import BaseAutotuner
from helion.runtime.config import Config
from helion.runtime.settings import default_autotuner_fn
# TODO(gmagogsfm): Remove CustomOp fallback path (_get_or_register_custom_op,
# vllm_helion_lib, direct_register_custom_op) once vLLM requires PyTorch >= 2.11.
# FIXME(gmagogsfm): Re-enable HOP path once performance regression is fixed.
# _HOP_AVAILABLE = requires_torch_version("2.11")
_HOP_AVAILABLE = False
if _HOP_AVAILABLE:
from helion._compat import supports_torch_compile_fusion
from helion._compiler._dynamo.higher_order_ops import helion_kernel_side_table
from helion._compiler._dynamo.variables import HelionKernelVariable
from helion.runtime.kernel import Kernel
from torch._dynamo.guards import GuardBuilder
from torch._dynamo.variables.builder import VariableBuilder
logger = init_logger(__name__)
vllm_helion_lib = Library("vllm_helion", "FRAGMENT") # noqa
ConfigPicker = Callable[[tuple[Any, ...], list[CaseKey]], CaseKey | None]
def validate_helion_settings(
helion_settings: helion.Settings | None, op_name: str
) -> None:
if helion_settings is None:
return
settings_dict = helion_settings.to_dict()
if (
"autotuner_fn" in settings_dict
and settings_dict["autotuner_fn"] is not None
and settings_dict["autotuner_fn"] is not default_autotuner_fn
):
raise ValueError(
f"HelionKernelWrapper for '{op_name}' uses a custom autotuner via "
f"config picker. Remove 'autotuner_fn' from helion_settings and use "
f"register_kernel(..., config_picker=...) instead."
)
if settings_dict.get("static_shapes") is True:
logger.warning(
"Kernel '%s' has static_shapes=True in helion_settings, "
"which will be overridden to False. vLLM requires dynamic "
"shapes for variable batch sizes and sequence lengths.",
op_name,
)
def create_helion_decorated_kernel(
raw_kernel_func: Callable,
helion_settings: helion.Settings | None = None,
extra_kwargs: dict[str, Any] | None = None,
) -> Any:
kernel_kwargs: dict[str, Any] = {}
if helion_settings:
kernel_kwargs.update(helion_settings.to_dict())
# vLLM requires dynamic shapes for variable batch sizes and sequence lengths
kernel_kwargs["static_shapes"] = False
if extra_kwargs:
kernel_kwargs.update(extra_kwargs)
return helion.kernel(**kernel_kwargs)(raw_kernel_func)
class PresetConfigSearch(BaseAutotuner):
"""Custom autotuner that uses a preset config selector instead of autotuning."""
def __init__(
self,
args: tuple[Any, ...],
config_selector: Callable[[tuple[Any, ...]], Config],
):
self.args = args
self.config_selector = config_selector
def autotune(self, *, skip_cache: bool = False) -> Config:
return self.config_selector(self.args)
class ConfiguredHelionKernel:
"""A configured Helion kernel bound to a specific platform."""
def __init__(
self,
op_name: str,
config_picker: ConfigPicker | None,
raw_kernel_func: Callable,
helion_settings: helion.Settings | None = None,
):
self.op_name = op_name
self.config_picker = config_picker
self.raw_kernel_func = raw_kernel_func
self.helion_settings = helion_settings
self._decorated_kernel = self._create_decorated_kernel()
def __call__(self, *args, **kwargs):
return self._decorated_kernel(*args, **kwargs)
def _create_key_computer(self):
"""Create a key computer function derived from the config picker.
The returned function receives kernel arguments unpacked (*args) to match
Helion's key signature (called as self._key_fn(*args)).
"""
if self.config_picker is None:
raise RuntimeError(
f"No config picker registered for kernel '{self.op_name}'. "
f"A config_picker must be provided to register_kernel()."
)
picker = self.config_picker
all_keys = list(self.configs.keys())
default = CaseKey.default()
has_default = default in self.configs
def key_computer(*args):
selected = picker(args, all_keys)
if selected is not None:
return str(selected)
if has_default:
return str(default)
return None
return key_computer
def _create_config_selector(self, key_computer):
str_to_key = {str(k): k for k in self.configs}
def config_selector(args):
selected_str = key_computer(*args)
if selected_str is None:
raise ValueError(
f"Config picker returned None for kernel "
f"'{self.op_name}' with available config keys: "
f"{list(self.configs.keys())}"
)
config_key = str_to_key.get(selected_str)
if config_key is None:
raise ValueError(
f"Config picker returned invalid config key "
f"'{selected_str}' for kernel "
f"'{self.op_name}'. "
f"Available keys: {list(self.configs.keys())}"
)
return self.configs[config_key]
return config_selector
def _load_platform_configs(self) -> None:
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.utils import get_canonical_gpu_name
self.platform = get_canonical_gpu_name()
config_manager = ConfigManager()
self.configs = config_manager.get_platform_configs(self.op_name, self.platform)
if not self.configs:
raise ValueError(
f"No configs available for kernel '{self.op_name}' "
f"on platform '{self.platform}'"
)
def _create_decorated_kernel(self) -> Callable[..., Any]:
self._load_platform_configs()
key_computer = self._create_key_computer()
config_selector = self._create_config_selector(key_computer)
extra_kwargs = {
"autotuner_fn": lambda _, args: PresetConfigSearch(args, config_selector),
"key": key_computer,
}
logger.debug(
"Creating decorated kernel %s with custom autotuner on platform %s",
self.op_name,
self.platform,
)
return create_helion_decorated_kernel(
self.raw_kernel_func, self.helion_settings, extra_kwargs
)
class HelionKernelWrapper:
"""Wrapper for Helion kernels with pre-tuned config selection and HOP support."""
def __init__(
self,
raw_kernel_func: Callable,
op_name: str,
fake_impl: Callable | None,
config_picker: ConfigPicker,
mutates_args: list[str] | None = None,
helion_settings: helion.Settings | None = None,
input_generator: (Callable[[], dict[CaseKey, tuple[Any, ...]]] | None) = None,
):
# Validate helion_settings doesn't conflict with our custom autotuner
validate_helion_settings(helion_settings, op_name)
self.raw_kernel_func = raw_kernel_func
self.op_name = op_name
self._fake_impl = fake_impl
self.helion_settings = helion_settings
self._config_picker = config_picker
self._input_generator = input_generator
self._mutates_args = mutates_args
self._configured_kernel: ConfiguredHelionKernel | None = None
# TODO(@gmagogsfm): Remove this disable flag once integrated with vLLM IR,
# which handles op enablement/disablement.
self._disabled = False
self._disabled_reason: str | None = None
try:
if not _HOP_AVAILABLE:
self._get_or_register_custom_op()
else:
self.get_configured_op()
except ValueError as e:
self._disabled = True
self._disabled_reason = str(e)
logger.warning(
"Helion kernel '%s' is disabled: %s",
op_name,
self._disabled_reason,
)
def __call__(self, *args, **kwargs):
if self._disabled:
raise RuntimeError(
f"Helion kernel '{self.op_name}' is disabled: {self._disabled_reason}"
)
if not _HOP_AVAILABLE:
op = getattr(torch.ops.vllm_helion, self.op_name)
return op(*args, **kwargs)
assert self._configured_kernel is not None, (
f"Kernel '{self.op_name}' was not initialized. "
"Please open an issue on GitHub."
)
# During Dynamo tracing, this call will be intercepted by our custom
# HelionKernelWrapperVariable and handled via proper HOP emission.
# During eager execution, call the kernel directly.
return self._configured_kernel(*args, **kwargs)
def get_inputs(self) -> dict[CaseKey, tuple[Any, ...]]:
if self._input_generator is None:
raise NotImplementedError(
f"No input generator registered for kernel '{self.op_name}'. "
f"Use register_kernel(..., input_generator=...) to register one."
)
return self._input_generator()
def run_autotune(
self,
inputs: tuple[Any, ...],
autotune_effort: str = "quick",
) -> Config:
"""Run autotuning for a single input configuration."""
extra_kwargs = {
"autotune_effort": autotune_effort,
"autotune_ignore_errors": True,
}
autotune_kernel = create_helion_decorated_kernel(
self.raw_kernel_func, self.helion_settings, extra_kwargs
)
return autotune_kernel.autotune(inputs)
def get_configured_op(self) -> ConfiguredHelionKernel:
if self._disabled:
raise RuntimeError(
f"Helion kernel '{self.op_name}' is disabled: {self._disabled_reason}"
)
if self._configured_kernel is None:
self._configured_kernel = ConfiguredHelionKernel(
op_name=self.op_name,
config_picker=self._config_picker,
raw_kernel_func=self.raw_kernel_func,
helion_settings=self.helion_settings,
)
return self._configured_kernel
def _get_or_register_custom_op(self) -> Any:
if hasattr(torch.ops.vllm_helion, self.op_name):
return getattr(torch.ops.vllm_helion, self.op_name)
configured_kernel = self.get_configured_op()
logger.info("Registering op: vllm_helion::%s", self.op_name)
direct_register_custom_op(
op_name=self.op_name,
op_func=configured_kernel._decorated_kernel,
mutates_args=self._mutates_args,
fake_impl=self._fake_impl,
target_lib=vllm_helion_lib,
)
return getattr(torch.ops.vllm_helion, self.op_name)
# Global registry for tracking all registered HelionKernelWrapper instances
_REGISTERED_KERNELS: dict[str, HelionKernelWrapper] = {}
def get_registered_kernels() -> dict[str, HelionKernelWrapper]:
return _REGISTERED_KERNELS.copy()
def get_kernel_by_name(kernel_name: str) -> HelionKernelWrapper | None:
return _REGISTERED_KERNELS.get(kernel_name)
def infer_fake_impl(
kernel_func: Callable,
helion_settings: helion.Settings | None = None,
) -> Callable:
def helion_fake_kernel(*args, **kwargs):
kernel_kwargs = {}
if helion_settings:
kernel_kwargs.update(helion_settings.to_dict())
temp_decorated_kernel = helion.kernel(**kernel_kwargs)(kernel_func)
# Bind with args to get config_spec, then get a valid default config
bound = temp_decorated_kernel.bind(args)
default_config = bound.config_spec.default_config()
compiled_runner = bound.compile_config(default_config)
return compiled_runner(*args, **kwargs, _launcher=lambda *a, **kw: None)
return helion_fake_kernel
def register_kernel(
op_name: str | None = None,
*,
config_picker: ConfigPicker,
fake_impl: Callable | None = None,
mutates_args: list[str] | None = None,
helion_settings: helion.Settings | None = None,
input_generator: (Callable[[], dict[CaseKey, tuple[Any, ...]]] | None) = None,
) -> Callable[[Callable], HelionKernelWrapper]:
"""Register a Helion kernel with pre-tuned config selection.
Args:
op_name: Name to register the op under. Defaults to the function name.
fake_impl: Optional meta/fake implementation for tracing.
mutates_args: Names of arguments the kernel mutates in place.
helion_settings: Optional Helion settings override.
config_picker: Required. Receives ``(args, config_keys)``
where each config key is a ``dict[str, Any]`` mapping
parameter names to values. Return the best-matching
dict, or ``None`` to fall back to the default config.
Example::
def pick_config(args, config_keys):
x = args[0]
best = min(config_keys, key=lambda k: abs(k["size"] - x.shape[0]))
return best
input_generator: Optional. Returns ``dict[str, tuple]`` where
each key is a serialized config key and each value is a
tuple of arguments to pass to the kernel.
Example::
def generate_inputs():
return {
"4096": (torch.randn(4096, device="cuda"), 0.5),
"8192": (torch.randn(8192, device="cuda"), 0.5),
}
"""
def decorator(kernel_func: Callable) -> HelionKernelWrapper:
final_op_name = op_name if op_name else kernel_func.__name__
if final_op_name in _REGISTERED_KERNELS:
raise ValueError(
f"Helion kernel '{final_op_name}' is already registered. "
f"Use a different op_name or check for duplicate registrations."
)
# A void-returning kernel that mutates its args needs no fake impl:
# PyTorch auto-generates a trivial one for mutable, no-return schemas.
returns_none = inspect.signature(kernel_func).return_annotation is None
final_fake_impl = fake_impl
if final_fake_impl is None and not (mutates_args and returns_none):
final_fake_impl = infer_fake_impl(kernel_func, helion_settings)
logger.debug(
"Auto-generated fake_impl for Helion kernel '%s'",
kernel_func.__name__,
)
kernel_wrapper = HelionKernelWrapper(
raw_kernel_func=kernel_func,
op_name=final_op_name,
fake_impl=final_fake_impl,
config_picker=config_picker,
mutates_args=mutates_args,
helion_settings=helion_settings,
input_generator=input_generator,
)
_REGISTERED_KERNELS[final_op_name] = kernel_wrapper
logger.info(
"Registered Helion kernel '%s' as HelionKernelWrapper",
kernel_func.__name__,
)
return kernel_wrapper
return decorator
# Register HelionKernelWrapper with Dynamo's variable tracker system
if _HOP_AVAILABLE:
def _register_vllm_helion_dynamo_variable():
"""Register HelionKernelWrapper with Dynamo's VariableBuilder.
When Dynamo encounters a HelionKernelWrapper during tracing, this
extracts the underlying Helion Kernel and delegates to Helion's own
registered Kernel handler, which handles HOP emission, side table
registration, and inductor lowering setup.
"""
def wrap_helion_kernel_wrapper(
builder: VariableBuilder, value: HelionKernelWrapper
):
kernel = value.get_configured_op()._decorated_kernel
if supports_torch_compile_fusion():
helion_handler = VariableBuilder._type_dispatch()[Kernel]
return helion_handler(builder, kernel)
kernel_idx = helion_kernel_side_table.add_kernel(kernel)
builder.install_guards(GuardBuilder.ID_MATCH)
return HelionKernelVariable(kernel, kernel_idx, source=builder.source)
dispatch = VariableBuilder._type_dispatch()
dispatch[HelionKernelWrapper] = wrap_helion_kernel_wrapper
# Register immediately when the module is imported
_register_vllm_helion_dynamo_variable()
-93
View File
@@ -1,93 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility functions for Helion kernel management."""
import regex as re
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
logger = init_logger(__name__)
# Maps known variant GPU names (after lowercase/underscore normalization)
# to their canonical form.
#
# Names that are already canonical after normalization are NOT listed here.
# For example, "NVIDIA H200" normalizes to "nvidia_h200" which needs no
# further mapping, and AMD ROCm names like "AMD_Instinct_MI300X" come from
# a controlled lookup table in rocm.py and normalize cleanly to
# "amd_instinct_mi300x". Only names with variant suffixes (form factor,
# memory size, memory type, etc.) that should be stripped need entries.
#
# To add a new GPU variant: run `canonicalize_gpu_name()` without the alias
# to see the normalized name, then add a mapping here if it contains variant
# suffixes that should be stripped (e.g. Blackwell/Rubin variants).
_GPU_NAME_ALIASES: dict[str, str] = {
# H100 variants
"nvidia_h100_pcie": "nvidia_h100",
"nvidia_h100_sxm5": "nvidia_h100",
"nvidia_h100_80gb_hbm3": "nvidia_h100",
"nvidia_h100_nvl": "nvidia_h100",
# H200 variants
"nvidia_h200_nvl": "nvidia_h200",
"nvidia_h200_141gb_hbm3e": "nvidia_h200",
# A100 variants
"nvidia_a100_sxm4_80gb": "nvidia_a100",
"nvidia_a100_sxm4_40gb": "nvidia_a100",
"nvidia_a100_pcie_80gb": "nvidia_a100",
"nvidia_a100_pcie_40gb": "nvidia_a100",
"nvidia_a100_80gb_pcie": "nvidia_a100",
# V100 variants (Tesla-branded)
"tesla_v100_sxm2_32gb": "tesla_v100",
"tesla_v100_sxm2_16gb": "tesla_v100",
"tesla_v100_pcie_32gb": "tesla_v100",
"tesla_v100_pcie_16gb": "tesla_v100",
# AMD ROCm variants (from _ROCM_DEVICE_ID_NAME_MAP in rocm.py)
"amd_instinct_mi300x_hf": "amd_instinct_mi300x",
# ADD MORE HERE
}
def get_gpu_name(device_id: int | None = None) -> str:
if device_id is None:
logger.warning_once(
"get_gpu_name() called without device_id, defaulting to 0. "
"This may return the wrong device name in multi-node setups."
)
device_id = 0
return current_platform.get_device_name(device_id)
def canonicalize_gpu_name(name: str) -> str:
"""Canonicalize GPU name for use as a platform identifier.
Converts to lowercase, replaces separators with underscores,
and maps known variant names to their canonical form via _GPU_NAME_ALIASES.
e.g., "NVIDIA H100 80GB HBM3" -> "nvidia_h100"
"NVIDIA A100-SXM4-80GB" -> "nvidia_a100"
"AMD Instinct MI300X" -> "amd_instinct_mi300x"
"""
if not name or not name.strip():
raise ValueError("GPU name cannot be empty")
name = re.sub(r"[\s/-]+", "_", name.lower())
if name in _GPU_NAME_ALIASES:
return _GPU_NAME_ALIASES[name]
return name
def get_canonical_gpu_name(device_id: int | None = None) -> str:
return canonicalize_gpu_name(get_gpu_name(device_id))
def get_fp8_dtype() -> torch.dtype:
return current_platform.fp8_dtype()
def get_int8_min_max() -> tuple[int, int]:
qtype_traits = torch.iinfo(torch.int8)
return qtype_traits.min, qtype_traits.max
def get_int8_min_scaling_factor() -> float:
return torch.finfo(torch.float32).eps
-127
View File
@@ -1,127 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""This file registers Oink implementations for vLLM IR ops.
vLLM does not depend on the external Oink repository/package. When an external
plugin registers torch.library.custom_op entrypoints under the `oink::`
namespace (e.g. via vLLM's general_plugins mechanism), these ops will be marked
as supported. To dispatch to those ops, set kernel_config.ir_op_priority.<op> to oink.
Alternatively, `VLLM_USE_OINK_OPS=1` will add this to priority by default.
"""
import torch
from torch import Tensor
from vllm import ir
from vllm.platforms import current_platform
OINK_AVAILABLE = current_platform.has_device_capability(100) and hasattr(
torch.ops, "oink"
)
def has_oink_op(name: str) -> bool:
"""Check if a specific oink op is registered."""
return OINK_AVAILABLE and hasattr(torch.ops.oink, name)
def _can_view_as_2d(x: Tensor) -> bool:
"""Return True if x.view(-1, x.shape[-1]) is viewable (no copy)."""
if x.dim() < 2:
return False
if x.dim() == 2:
return True
# For a view(-1, N) to be valid, all leading dims must be contiguous with
# respect to each other (size-1 dims are ignored).
for dim in range(x.dim() - 1):
# Strides for size-1 dims are irrelevant and can be arbitrary.
if x.size(dim + 1) != 1 and x.stride(dim) != x.stride(dim + 1) * x.size(
dim + 1
):
return False
return True
def _is_oink_stride_compatible_2d(x_2d: Tensor) -> bool:
"""Return True if x_2d meets Oink's pointer-path stride constraints."""
if x_2d.dim() != 2:
return False
if x_2d.stride(1) != 1:
return False
# Match Oink's vectorization constraint: stride(0) divisible by 256b.
if x_2d.dtype in (torch.float16, torch.bfloat16):
divby = 16
elif x_2d.dtype == torch.float32:
divby = 8
else:
return False
return (x_2d.stride(0) % divby) == 0
oink_rms_supported = (
lambda x, weight, epsilon, variance_size=None: variance_size is None
and weight is not None
and x.dim() >= 2
and x.dtype == weight.dtype
and weight.is_contiguous()
and _can_view_as_2d(x)
and _is_oink_stride_compatible_2d(x.view(-1, x.shape[-1]))
)
"""
Oink rms only supports 2d-like inputs with contiguous weight
and no variance_size override.
"""
@ir.ops.rms_norm.register_impl(
"oink", supports_args=oink_rms_supported, supported=has_oink_op("rmsnorm")
)
def rms_norm(
x: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> Tensor:
assert variance_size is None
x_2d = x.view(-1, x.shape[-1])
return torch.ops.oink.rmsnorm(x_2d, weight, epsilon).view_as(x)
oink_add_rms_supported = (
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
and weight is not None
and x.dim() >= 2
and x.dtype == weight.dtype
and weight.is_contiguous()
and _can_view_as_2d(x)
and _is_oink_stride_compatible_2d(x.view(-1, x.shape[-1]))
# residual must have 2d-compatible strides and match x shape/dtype
and x.dtype == x_residual.dtype
and x.shape == x_residual.shape
and _can_view_as_2d(x_residual)
and _is_oink_stride_compatible_2d(x_residual.view(-1, x_residual.shape[-1]))
)
"""
Oink fused_add_rms_norm has the same constraints as rms_norm,
and residual must be 2d-like with compatible strides.
"""
@ir.ops.fused_add_rms_norm.register_impl(
"oink",
supports_args=oink_add_rms_supported,
supported=has_oink_op("fused_add_rms_norm"),
inplace=True,
)
def fused_add_rms_norm(
x: Tensor,
x_residual: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> tuple[Tensor, Tensor]:
assert variance_size is None
x_2d = x.view(-1, x.shape[-1])
residual_2d = x_residual.view(-1, x_residual.shape[-1])
torch.ops.oink.fused_add_rms_norm(x_2d, residual_2d, weight, epsilon)
return x, x_residual
-6
View File
@@ -1,6 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.lora.ops.xpu_ops.lora_ops import bgmv_expand, bgmv_expand_slice, bgmv_shrink
__all__ = ["bgmv_expand", "bgmv_expand_slice", "bgmv_shrink"]
-148
View File
@@ -1,148 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.utils.torch_utils import direct_register_custom_op
def _bgmv_shrink_impl(
inputs: torch.Tensor,
lora_a_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
scaling: float,
) -> None:
torch.ops._xpu_C.bgmv_shrink(
output_tensor, inputs, lora_a_weights, lora_indices_tensor, scaling
)
def _bgmv_expand_impl(
inputs: torch.Tensor,
lora_b_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
add_inputs: bool,
) -> None:
weight_out_dim = lora_b_weights.size(-2)
output_dim = output_tensor.size(1)
if weight_out_dim == output_dim:
torch.ops._xpu_C.bgmv_expand(
output_tensor,
inputs,
lora_b_weights,
lora_indices_tensor,
add_inputs,
)
elif weight_out_dim < output_dim:
# LoRA weight output dim can be smaller than the output tensor
# (e.g. vocab_size vs padded logits). Use expand_slice to write
# only the matching portion, mirroring torch_ops common_len logic.
torch.ops._xpu_C.bgmv_expand_slice(
output_tensor,
inputs,
lora_b_weights,
lora_indices_tensor,
0,
weight_out_dim,
add_inputs,
)
else:
# Weight output dim larger than output tensor: truncate weights.
lora_b_weights = lora_b_weights[..., :output_dim, :].contiguous()
torch.ops._xpu_C.bgmv_expand_slice(
output_tensor,
inputs,
lora_b_weights,
lora_indices_tensor,
0,
output_dim,
add_inputs,
)
def _bgmv_expand_slice_impl(
inputs: torch.Tensor,
lora_b_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
slice_offset: int,
slice_size: int,
add_inputs: bool,
) -> None:
assert slice_size == lora_b_weights.size(-2)
assert slice_offset + slice_size <= output_tensor.size(1)
torch.ops._xpu_C.bgmv_expand_slice(
output_tensor,
inputs,
lora_b_weights,
lora_indices_tensor,
slice_offset,
slice_size,
add_inputs,
)
direct_register_custom_op(
op_name="xpu_bgmv_shrink",
op_func=_bgmv_shrink_impl,
mutates_args=["output_tensor"],
)
direct_register_custom_op(
op_name="xpu_bgmv_expand",
op_func=_bgmv_expand_impl,
mutates_args=["output_tensor"],
)
direct_register_custom_op(
op_name="xpu_bgmv_expand_slice",
op_func=_bgmv_expand_slice_impl,
mutates_args=["output_tensor"],
)
def bgmv_shrink(
inputs: torch.Tensor,
lora_a_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
scaling: float = 1.0,
) -> None:
torch.ops.vllm.xpu_bgmv_shrink(
inputs, lora_a_weights, output_tensor, lora_indices_tensor, scaling
)
def bgmv_expand(
inputs: torch.Tensor,
lora_b_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
add_inputs: bool = True,
) -> None:
torch.ops.vllm.xpu_bgmv_expand(
inputs, lora_b_weights, output_tensor, lora_indices_tensor, add_inputs
)
def bgmv_expand_slice(
inputs: torch.Tensor,
lora_b_weights: torch.Tensor,
output_tensor: torch.Tensor,
lora_indices_tensor: torch.Tensor,
slice_offset: int,
slice_size: int,
add_inputs: bool = True,
) -> None:
torch.ops.vllm.xpu_bgmv_expand_slice(
inputs,
lora_b_weights,
output_tensor,
lora_indices_tensor,
slice_offset,
slice_size,
add_inputs,
)
-766
View File
@@ -1,766 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Based on:
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
Punica: Multi-Tenant LoRA Serving.
https://arxiv.org/abs/2310.18547
"""
from typing import Any, final
import torch
from vllm import _custom_ops as ops
from vllm.lora.layers import LoRAMapping
from vllm.lora.ops.xpu_ops import bgmv_expand, bgmv_expand_slice, bgmv_shrink
from vllm.lora.utils import get_captured_lora_counts
from vllm.triton_utils import HAS_TRITON, triton
from vllm.utils.math_utils import round_up
if HAS_TRITON:
from vllm.lora.ops.triton_ops import (
LoRAKernelMeta,
fused_moe_lora,
)
from .punica_base import PunicaWrapperBase
@final
class PunicaWrapperXPU(PunicaWrapperBase):
"""PunicaWrapperXPU is designed to manage and provide metadata for the punica
kernel. The main function is to maintain the state information for
Multi-LoRA, and to provide the interface for the punica ipex kernel.
"""
def __init__(
self,
max_num_batched_tokens: int,
max_batches: int,
device: torch.device | str,
**kwargs,
):
PunicaWrapperBase.__init__(self, max_num_batched_tokens, max_batches, device)
torch._dynamo.mark_dynamic(self._token_lora_indices, 0)
torch._dynamo.mark_dynamic(self._embeddings_indices, 0)
torch._dynamo.mark_dynamic(self._sampler_indices_padded, 0)
self.lora_config = kwargs["lora_config"]
self.max_loras = self.lora_config.max_loras
# Compute captured LoRA counts for cudagraph specialization.
captured_lora_counts = get_captured_lora_counts(
self.max_loras, self.lora_config.specialize_active_lora
)
self.token_mapping_meta = LoRAKernelMeta.make(
self.max_loras,
max_num_batched_tokens,
device=device,
captured_lora_counts=captured_lora_counts,
)
# When speculative decoding is enabled, max_num_samples is
# max_batches * (num_speculative_decoding_tokens + 1).
# This line can be optimized by replacing max_num_batched_tokens
# to max_batches * (num_speculative_decoding_tokens + 1).
self.prompt_mapping_meta = LoRAKernelMeta.make(
self.max_loras,
max_num_batched_tokens,
device=device,
captured_lora_counts=captured_lora_counts,
)
def update_metadata(
self,
mapping: LoRAMapping,
lora_index_to_id: list[int | None],
max_loras: int,
vocab_size: int,
**kwargs,
):
self.is_prefill = mapping.is_prefill
self._update_base_metadata(mapping, lora_index_to_id, max_loras, vocab_size)
# Prepare kernel metadata tensors
self.token_mapping_meta.prepare_tensors(self.token_lora_indices)
self.prompt_mapping_meta.prepare_tensors(self.sampler_indices)
def _get_token_lora_indices(self, x: torch.Tensor) -> torch.IntTensor:
return torch.narrow(self._token_lora_indices, 0, 0, x.size(0))
def _apply_shrink(
self,
y: torch.Tensor,
x: torch.Tensor,
w_t_all: torch.Tensor,
scale: float,
):
buf = torch.zeros(
x.size(0),
w_t_all.size(-2),
dtype=x.dtype,
device=x.device,
)
bgmv_shrink(x, w_t_all, buf, self._get_token_lora_indices(x), scale)
y.copy_(buf)
def _apply_expand(
self,
y: torch.Tensor,
x: torch.Tensor,
w_t_all: torch.Tensor,
y_offset: int,
y_slice_size: int,
add_inputs: bool,
):
token_lora_indices = self._get_token_lora_indices(x)
# After tensor-parallel all-gather (non-fully-sharded LoRA), x may
# have been gathered along the rank dim so x.size(1) == max_lora_rank
# * tp_size, while lora_b only uses max_lora_rank elements. The XPU
# C++ kernel requires inputs.size(1) == lora_b.size(-1), so truncate
# to the actual rank. x[:, :rank] is non-contiguous, hence the copy.
rank = w_t_all.size(-1)
if x.size(1) != rank:
x = x[:, :rank].contiguous()
bgmv_expand_slice(
x, w_t_all, y, token_lora_indices, y_offset, y_slice_size, add_inputs
)
def add_shrink(
self,
y: torch.Tensor,
x: torch.Tensor,
lora_a_stacked: tuple[torch.Tensor, ...],
scale: float,
**kwargs: Any,
):
"""Performs GEMM for multiple slices of lora_a.
Semantics:
for i in range(len(lora_a_stacked)):
y[i] += (x @ lora_a_stacked[i]) * scale
Args:
y (torch.Tensor): Output tensors
x (torch.Tensor): Input tensor
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weights
scale (float): Scaling factor for the operation
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
x = x.view(-1, x.shape[-1])
for slice_idx in range(len(lora_a_stacked)):
self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx], scale)
def add_expand(
self,
y: torch.Tensor,
x: torch.Tensor,
lora_b_stacked: tuple[torch.Tensor, ...],
output_slices: tuple[int, ...],
offset_start: int = 0,
add_inputs=True,
**kwargs: Any,
) -> None:
"""Performs GEMM for multiple slices of lora_b.
Semantics:
for i in range(len(lora_b_stacked)):
slice = output_slices[i]
y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i]
offset += slice
Args:
y (torch.Tensor): Output tensor.
x (torch.Tensor): Input tensors
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
output_slices (tuple[int, ...]): Every slice's size
offset_start (int): The starting position of y, defaults to 0.
add_inputs (bool): Defaults to True.
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
y_org = y
y = y.view(-1, y.shape[-1])
assert x.ndim == 3
assert x.size(0) == len(output_slices)
# TODO fuse these kernels
for slice_idx in range(len(lora_b_stacked)):
self._apply_expand(
y,
x[slice_idx],
lora_b_stacked[slice_idx],
offset_start,
output_slices[slice_idx],
add_inputs=add_inputs,
)
offset_start += output_slices[slice_idx]
y = y.view_as(y_org)
def add_lora_embedding(
self,
y: torch.Tensor,
x: torch.Tensor,
lora_b_stacked: torch.Tensor,
add_inputs: bool = True,
**kwargs: Any,
) -> None:
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA.
Semantics:
y += x @ lora_b_stacked
Args:
y (torch.Tensor): Output tensor.
x (torch.Tensor): Input tensor.
lora_b_stacked (torch.Tensor): lora_b's weights.
add_inputs (bool): Default to True.
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
token_lora_indices = self._get_token_lora_indices(x)
bgmv_expand(x, lora_b_stacked, y, token_lora_indices, add_inputs)
def add_lora_linear(
self,
y: torch.Tensor,
x: torch.Tensor,
lora_a_stacked: tuple[torch.Tensor, ...],
lora_b_stacked: tuple[torch.Tensor, ...],
scale: float,
output_slices: tuple[int, ...],
*,
buffer: torch.Tensor | None = None,
**kwargs: Any,
) -> None:
"""Applicable to linear-related lora.
Semantics:
for i in range(len(lora_a_stacked)):
y[i] += (
x[i].unsqueeze(0)
@ lora_a_stacked[indices[i], layer_idx, :, :]
@ lora_b_stacked[indices[i], layer_idx, :, :]
* scale
).squeeze(0)
Args:
y (torch.Tensor): Output tensor. Will be changed in-place.
x (torch.Tensor): Input tensor
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weight.
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight.
scale (float): Scaling factor.
output_slices (tuple[int, ...]): Every slice's size.
buffer (Optional[torch.Tensor]): Defaults to None.
**kwargs: Accepts `add_inputs` (bool, defaults to True); any other
keys are ignored.
"""
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
assert buffer is None, (
"To minimize overhead, the buffer should be created by "
".add_lora_linear() instead of being passed in."
)
r = lora_b_stacked[0].size(-1)
buffer = torch.zeros( # type: ignore
(len(output_slices), x.size(0), r),
dtype=x.dtype,
device=x.device,
)
add_inputs = kwargs.pop("add_inputs", True)
self.add_shrink(
buffer, # type: ignore
x,
lora_a_stacked,
scale,
**kwargs,
)
self.add_expand(
y,
buffer, # type: ignore
lora_b_stacked,
output_slices,
add_inputs=add_inputs,
**kwargs,
)
@property
def sampler_indices_padded(self) -> torch.Tensor:
"""This property provides access to padded sampler indices."""
return self._sampler_indices_padded[:]
def add_lora_logits(
self,
y: torch.Tensor,
x: torch.Tensor,
lora_a_stacked: torch.Tensor,
lora_b_stacked: torch.Tensor,
scale,
*,
buffer: torch.Tensor | None = None,
**kwargs: Any,
) -> None:
"""Applies lora specifically for LogitsProcessorWithLoRA.
Semantics:
buffer = (x @ lora_a_stacked) * scale
y += buffer @ lora_b_stacked
Args:
y (torch.Tensor): Output tensor.
x (torch.Tensor): Input tensor.
lora_a_stacked (torch.Tensor): lora_a's weights.
lora_b_stacked (torch.Tensor): lora_b's weights.
scale (float): Scaling factor.
buffer (Optional[torch.Tensor]): Default to None.
**kwargs: Unused; accepted for compatibility with the base class signature.
"""
y_org = y
y = y.view(-1, y.shape[-1])
x = x.view(-1, x.shape[-1])
r = lora_b_stacked.size(-1)
assert buffer is None, (
"To minimize overhead, the buffer should be created by "
".add_lora_linear() instead of being passed in."
)
buffer = torch.zeros((x.size(0), r), dtype=x.dtype, device=x.device)
sampler_indices = torch.narrow(self._sampler_indices, 0, 0, x.size(0))
bgmv_shrink(x, lora_a_stacked, buffer, sampler_indices, scale)
bgmv_expand(buffer, lora_b_stacked, y, sampler_indices, add_inputs=True)
y = y.view_as(y_org)
def apply_lora_full_linear(
self,
y: torch.Tensor,
x: torch.Tensor,
weight_stacked: torch.Tensor,
bias_stacked: torch.Tensor,
module_enabled: torch.Tensor,
) -> None:
indices = self.sampler_indices
adapter_y = torch.zeros(
(x.size(0), weight_stacked.size(-2)),
dtype=torch.float32,
device=x.device,
)
bgmv_shrink(x, weight_stacked, adapter_y, indices, 1.0)
result = self._select_full_linear_output(
y, adapter_y, bias_stacked, module_enabled
)
y.copy_(result)
def moe_lora_align_block_size(
self,
topk_ids: torch.Tensor,
num_tokens: int,
block_size: int,
num_experts: int,
max_loras: int,
adapter_enabled: torch.Tensor,
expert_map: torch.Tensor | None = None,
pad_sorted_ids: bool = False,
naive_block_assignment: bool = False,
token_lora_mapping: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Aligns tokens and experts into block-sized chunks for LoRA-based
mixture-of-experts (MoE) execution.
When `token_lora_mapping` is provided, it overrides the global mapping
read from `self.token_mapping_meta`. This is how EP+LoRA injects the
per-rank-local token→LoRA map after all-to-all dispatch.
"""
(
token_lora_mapping_meta,
_,
_,
_,
lora_ids,
_,
_,
) = self.token_mapping_meta.meta_args(
num_tokens, self.lora_config.specialize_active_lora
)
if token_lora_mapping is None:
token_lora_mapping = token_lora_mapping_meta
# Under EP the caller passes local_num_experts but topk_ids carries
# GLOBAL expert indices. The CUDA kernel uses num_experts to size
# its bucketing table; with EP we must size by global_num_experts
# so global topk_ids don't overflow. expert_map inside the kernel
# then translates global→local so the output expert_ids are local
# (mirrors the non-LoRA moe_align_block_size behavior).
kernel_num_experts = (
expert_map.numel() if expert_map is not None else num_experts
)
if naive_block_assignment:
expert_ids = topk_ids.reshape(-1)
sorted_ids = None
num_tokens_post_pad = None
else:
max_num_tokens_padded = topk_ids.numel() + kernel_num_experts * (
block_size - 1
)
if pad_sorted_ids:
max_num_tokens_padded = round_up(max_num_tokens_padded, block_size)
if topk_ids.numel() < kernel_num_experts:
max_num_tokens_padded = topk_ids.numel() * block_size
sorted_ids = torch.empty(
(max_loras * max_num_tokens_padded,),
dtype=torch.int32,
device=topk_ids.device,
)
max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size)
# Expert ids are initialized to -1 so unused (lora, expert)
# slots don't drive the LoRA Triton kernel into the wrong bucket.
# The kernel overwrites only active slots.
expert_ids = torch.full(
(max_loras * max_num_m_blocks,),
-1,
dtype=torch.int32,
device=topk_ids.device,
)
num_tokens_post_pad = torch.empty(
(max_loras), dtype=torch.int32, device=topk_ids.device
)
ops.moe_lora_align_block_size(
topk_ids,
token_lora_mapping,
kernel_num_experts,
block_size,
max_loras,
max_num_tokens_padded,
max_num_m_blocks,
sorted_ids,
expert_ids,
num_tokens_post_pad,
adapter_enabled,
lora_ids,
expert_map,
)
return token_lora_mapping, sorted_ids, expert_ids, num_tokens_post_pad
def add_lora_fused_moe(
self,
y: torch.Tensor,
x: torch.Tensor,
lora_a_stacked: tuple[torch.Tensor, ...],
lora_b_stacked: tuple[torch.Tensor, ...],
topk_weights: torch.Tensor,
sorted_token_ids: torch.Tensor | None,
expert_ids: torch.Tensor,
num_tokens_post_padded: torch.Tensor | None,
max_lora_rank: int,
top_k_num: int,
shrink_config,
expand_config,
adapter_enabled: torch.Tensor,
mul_routed_weight=False,
fully_sharded: bool = False,
offset: int = 0,
token_lora_mapping: torch.Tensor | None = None,
add_inputs: bool = True,
):
"""Perform a fused forward computation for a LoRA MoE layer."""
(
token_lora_mapping_meta,
_,
_,
_,
lora_ids,
no_lora_flag,
num_active_loras,
) = self.token_mapping_meta.meta_args(
x.size(0), self.lora_config.specialize_active_lora
)
assert no_lora_flag.numel() == 1
if no_lora_flag.item():
# None of the inputs require LoRA.
return
if token_lora_mapping is None:
token_lora_mapping = token_lora_mapping_meta
fused_moe_lora(
y,
x,
lora_a_stacked,
lora_b_stacked,
topk_weights,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
token_lora_mapping,
max_lora_rank,
top_k_num,
lora_ids,
num_active_loras,
adapter_enabled,
shrink_config.get("BLOCK_SIZE_M", 64),
shrink_config.get("BLOCK_SIZE_N", 64),
shrink_config.get("BLOCK_SIZE_K", 32),
shrink_config.get("GROUP_SIZE_M", 8),
shrink_config.get("NUM_WARPS", 4),
shrink_config.get("NUM_STAGES", 3),
shrink_config.get("SPLIT_K", 1),
expand_config.get("BLOCK_SIZE_M", 64),
expand_config.get("BLOCK_SIZE_N", 64),
expand_config.get("BLOCK_SIZE_K", 32),
expand_config.get("GROUP_SIZE_M", 8),
expand_config.get("NUM_WARPS", 4),
expand_config.get("NUM_STAGES", 3),
expand_config.get("SPLIT_K", 1),
mul_routed_weight,
fully_sharded,
offset,
add_inputs,
)
def add_lora_w13(
self,
y: torch.Tensor,
x: torch.Tensor,
lora_a_stacked: tuple[torch.Tensor, ...],
lora_b_stacked: tuple[torch.Tensor, ...],
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
expert_map: torch.Tensor | None,
w1: torch.Tensor,
w2: torch.Tensor,
num_tokens: int,
top_k_num: int,
max_loras: int,
adapter_enabled: torch.Tensor,
local_num_experts: int,
top_k: int,
num_slices: int,
fully_sharded: bool,
use_tuned_config: bool,
add_inputs: bool = True,
token_lora_mapping: torch.Tensor | None = None,
) -> tuple[
torch.Tensor | None,
torch.Tensor | None,
torch.Tensor | None,
torch.Tensor | None,
]:
import functools
from vllm.lora.layers.utils import try_get_optimal_moe_lora_config
from vllm.lora.ops.triton_ops.utils import (
_normalize_lora_config_keys,
get_lora_op_configs,
)
from vllm.model_executor.layers.fused_moe.config import _get_config_dtype_str
config_dtype = _get_config_dtype_str(
dtype=x.dtype,
use_fp8_w8a8=False,
use_int8_w8a16=False,
use_int4_w4a16=False,
)
max_lora_rank = lora_a_stacked[0].shape[-2]
if use_tuned_config:
shrink_config = get_lora_op_configs(
op_type="fused_moe_lora_w13_shrink",
max_loras=max_loras,
batch=num_tokens,
hidden_size=x.shape[-1],
rank=max_lora_rank,
num_slices=num_slices,
moe_intermediate_size=lora_b_stacked[0].shape[-2],
)
expand_config = get_lora_op_configs(
op_type="fused_moe_lora_w13_expand",
max_loras=max_loras,
batch=num_tokens,
hidden_size=x.shape[-1],
rank=max_lora_rank,
num_slices=num_slices,
moe_intermediate_size=lora_b_stacked[0].shape[-2],
)
else:
get_config = functools.partial(
try_get_optimal_moe_lora_config,
w1_shape=w1.shape,
w2_shape=w2.shape,
rank=max_lora_rank,
top_k=top_k,
dtype=config_dtype,
M=num_tokens,
)
shrink_config = get_config(op_type="fused_moe_lora_w13_shrink")
expand_config = get_config(op_type="fused_moe_lora_w13_expand")
shrink_config = _normalize_lora_config_keys(shrink_config)
expand_config = _normalize_lora_config_keys(expand_config)
SPARSITY_FACTOR = 8
naive_block_assignment = (
not fully_sharded
and expert_map is None
and num_tokens * top_k * SPARSITY_FACTOR <= local_num_experts * max_loras
)
(
token_lora_mapping,
sorted_token_ids_lora,
expert_ids_lora,
num_tokens_post_padded_lora,
) = self.moe_lora_align_block_size(
topk_ids,
num_tokens,
int(shrink_config.get("BLOCK_SIZE_M") or 64),
local_num_experts,
max_loras,
adapter_enabled,
expert_map,
naive_block_assignment=naive_block_assignment,
token_lora_mapping=token_lora_mapping,
)
_sorted = sorted_token_ids_lora
_eids = expert_ids_lora
if _sorted is not None:
_eids = _eids.view(max_loras, -1)
_sorted = _sorted.view(max_loras, -1)
self.add_lora_fused_moe(
y.view(-1, top_k_num, y.shape[-1]),
x,
lora_a_stacked,
lora_b_stacked,
topk_weights,
_sorted,
_eids,
num_tokens_post_padded_lora,
max_lora_rank,
top_k,
shrink_config,
expand_config,
adapter_enabled,
fully_sharded=fully_sharded,
token_lora_mapping=token_lora_mapping,
add_inputs=add_inputs,
)
return (
sorted_token_ids_lora,
expert_ids_lora,
num_tokens_post_padded_lora,
token_lora_mapping,
)
def add_lora_w2(
self,
y: torch.Tensor,
x: torch.Tensor,
lora_a_stacked: tuple[torch.Tensor, ...],
lora_b_stacked: tuple[torch.Tensor, ...],
topk_weights: torch.Tensor,
sorted_token_ids_lora: torch.Tensor | None,
expert_ids_lora: torch.Tensor | None,
num_tokens_post_padded_lora: torch.Tensor | None,
token_lora_mapping: torch.Tensor | None,
num_tokens: int,
w1: torch.Tensor,
w2: torch.Tensor,
top_k_num: int,
max_loras: int,
adapter_enabled: torch.Tensor,
top_k: int,
fully_sharded: bool,
tp_rank: int,
use_tuned_config: bool,
add_inputs: bool = True,
) -> None:
import functools
from vllm.lora.layers.utils import try_get_optimal_moe_lora_config
from vllm.lora.ops.triton_ops.utils import (
_normalize_lora_config_keys,
get_lora_op_configs,
)
from vllm.model_executor.layers.fused_moe.config import _get_config_dtype_str
config_dtype = _get_config_dtype_str(
dtype=x.dtype,
use_fp8_w8a8=False,
use_int8_w8a16=False,
use_int4_w4a16=False,
)
max_lora_rank = lora_a_stacked[0].shape[-2]
if use_tuned_config:
shrink_config = get_lora_op_configs(
op_type="fused_moe_lora_w2_shrink",
max_loras=max_loras,
batch=num_tokens,
hidden_size=y.shape[-1],
rank=max_lora_rank,
num_slices=1,
moe_intermediate_size=lora_a_stacked[0].shape[-1],
)
expand_config = get_lora_op_configs(
op_type="fused_moe_lora_w2_expand",
max_loras=max_loras,
batch=num_tokens,
hidden_size=y.shape[-1],
rank=max_lora_rank,
num_slices=1,
moe_intermediate_size=lora_a_stacked[0].shape[-1],
)
else:
get_config = functools.partial(
try_get_optimal_moe_lora_config,
w1_shape=w1.shape,
w2_shape=w2.shape,
rank=max_lora_rank,
top_k=top_k,
dtype=config_dtype,
M=num_tokens,
)
shrink_config = get_config(op_type="fused_moe_lora_w2_shrink")
expand_config = get_config(op_type="fused_moe_lora_w2_expand")
shrink_config = _normalize_lora_config_keys(shrink_config)
expand_config = _normalize_lora_config_keys(expand_config)
_sorted = sorted_token_ids_lora
_eids = expert_ids_lora
if _sorted is not None:
assert _eids is not None
_eids = _eids.view(max_loras, -1)
_sorted = _sorted.view(max_loras, -1)
# w2_lora_b shape[-2] is hidden_size // tp_size when fully_sharded
shard_size = lora_b_stacked[0].shape[-2]
offset = shard_size * tp_rank if fully_sharded else 0
self.add_lora_fused_moe(
y,
x,
lora_a_stacked,
lora_b_stacked,
topk_weights,
_sorted,
_eids,
num_tokens_post_padded_lora,
max_lora_rank,
top_k,
shrink_config,
expand_config,
adapter_enabled,
True, # mul_routed_weight
fully_sharded=fully_sharded,
offset=offset,
token_lora_mapping=token_lora_mapping,
add_inputs=add_inputs,
)
@@ -1,602 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings.driver import CUstream
from cutlass import Float32, Int32, Uint32, Uint64
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import recast_val
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
)
from vllm.triton_utils import tl, triton
def _dcp_merge_topk_and_interleave(vllm_config: Any) -> tuple[int, int]:
"""(topk, cp_interleave) the DCP merge runs with.
Kpool indexers (GLM-5.x) merge pool ids: topk and the interleave are both
in units of ``index_kpool`` tokens.
"""
hf_text_config = vllm_config.model_config.hf_text_config
topk = hf_text_config.index_topk
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
kpool = getattr(hf_text_config, "index_kpool", None) or 1
if kpool > 1:
topk //= kpool
cp_interleave //= kpool
return topk, cp_interleave
def stable_topk_from_gathered_candidates_cutedsl(
gathered: torch.Tensor,
topk: int,
out: torch.Tensor | None = None,
) -> torch.Tensor:
if out is None:
out = torch.empty(
(gathered.shape[0], topk),
dtype=torch.int32,
device=gathered.device,
)
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL(gathered, out, topk=topk)
return out
def pack_dcp_topk_candidates_cutedsl(
logits: torch.Tensor,
topk_indices: torch.Tensor,
packed: torch.Tensor,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
row_starts: torch.Tensor | None,
) -> None:
topk = topk_indices.shape[1]
row_starts_arg = row_starts if row_starts is not None else topk_indices
_PACK_DCP_TOPK_CANDIDATES_KERNEL(
logits,
topk_indices,
packed,
row_starts_arg,
logits_stride0=logits.stride(0),
logits_stride1=logits.stride(1),
topk_stride0=topk_indices.stride(0),
topk_stride1=topk_indices.stride(1),
packed_stride0=packed.stride(0),
packed_stride1=packed.stride(1),
packed_stride2=packed.stride(2),
num_cols=logits.shape[1],
dcp_rank=dcp_rank,
dcp_world_size=dcp_world_size,
cp_interleave=cp_interleave,
has_row_starts=row_starts is not None,
topk=topk,
block_size=512,
)
class PackDCPTopkCandidatesKernel(
VllmTritonJitKernel["PackDCPTopkCandidatesKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
dcp_rank: int
dcp_world_size: int
cp_interleave: int
has_row_starts: bool
topk: int
block_size: int
# These scalars only describe runtime layouts and bounds; the constexpr
# fields below own the launch geometry and algorithmic specialization.
@staticmethod
@triton.jit(
do_not_specialize=[
"logits_stride0",
"logits_stride1",
"topk_stride0",
"topk_stride1",
"packed_stride0",
"packed_stride1",
"packed_stride2",
"num_cols",
]
)
def kernel(
logits,
topk_indices,
packed,
row_starts,
logits_stride0,
logits_stride1,
topk_stride0,
topk_stride1,
packed_stride0,
packed_stride1,
packed_stride2,
num_cols,
DCP_RANK: tl.constexpr,
DCP_WORLD_SIZE: tl.constexpr,
CP_INTERLEAVE: tl.constexpr,
HAS_ROW_STARTS: tl.constexpr,
TOPK: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
tile = tl.program_id(1)
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = cols < TOPK
local_idx = tl.load(
topk_indices + row * topk_stride0 + cols * topk_stride1,
mask=mask,
other=-1,
)
valid = local_idx >= 0
safe_local_idx = tl.maximum(local_idx, 0)
row_start = 0
if HAS_ROW_STARTS:
row_start = tl.load(row_starts + row)
score_col = safe_local_idx + row_start
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
score = tl.load(
logits + row * logits_stride0 + score_col * logits_stride1,
mask=mask & valid,
other=-float("inf"),
)
global_id = (
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
+ DCP_RANK * CP_INTERLEAVE
+ safe_local_idx % CP_INTERLEAVE
)
global_id = tl.where(valid, global_id, -1)
packed_base = packed + row * packed_stride0 + cols * packed_stride1
tl.store(packed_base, score, mask=mask)
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
def dispatch( # type: ignore[override]
self,
*,
has_row_starts: bool,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
has_row_starts=has_row_starts,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
topk, cp_interleave = _dcp_merge_topk_and_interleave(vllm_config)
if topk <= 0:
return []
try:
from vllm.distributed.parallel_state import get_dcp_group
dcp_rank = get_dcp_group().rank_in_group
except Exception:
dcp_rank = 0
return self._trace_dispatch(self.dispatch)(
dcp_rank=dcp_rank,
dcp_world_size=dcp_world_size,
cp_interleave=cp_interleave,
has_row_starts=(False, True),
topk=topk,
block_size=512,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
fp32_ptr = TritonWarmupTensor(torch.float32)
int32_ptr = TritonWarmupTensor(torch.int32)
return dict(
logits=fp32_ptr,
topk_indices=TritonWarmupTensor(torch.int32, shape=(1, compile_key.topk)),
packed=fp32_ptr,
row_starts_arg=int32_ptr,
logits_stride0=1,
logits_stride1=1,
topk_stride0=1,
topk_stride1=1,
packed_stride0=1,
packed_stride1=1,
packed_stride2=1,
num_cols=1,
dcp_rank=compile_key.dcp_rank,
dcp_world_size=compile_key.dcp_world_size,
cp_interleave=compile_key.cp_interleave,
has_row_starts=compile_key.has_row_starts,
topk=compile_key.topk,
block_size=compile_key.block_size,
)
@kernel_launcher
def __call__(
self,
logits: torch.Tensor,
topk_indices: torch.Tensor,
packed: torch.Tensor,
row_starts_arg: torch.Tensor,
*,
logits_stride0: int,
logits_stride1: int,
topk_stride0: int,
topk_stride1: int,
packed_stride0: int,
packed_stride1: int,
packed_stride2: int,
num_cols: int,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
has_row_starts: bool,
topk: int,
block_size: int,
) -> LaunchSpec:
grid = (topk_indices.shape[0], triton.cdiv(topk, block_size))
return grid, dict(
row_starts=row_starts_arg,
DCP_RANK=dcp_rank,
DCP_WORLD_SIZE=dcp_world_size,
CP_INTERLEAVE=cp_interleave,
HAS_ROW_STARTS=has_row_starts,
TOPK=topk,
BLOCK_SIZE=block_size,
num_warps=8,
)
class StableTopKFromGatheredCandidatesKernel(
VllmCuTeDSLJitKernel["StableTopKFromGatheredCandidatesKernel.CompileKey"]
):
tb_size = 512
hist_bins = 2048
radix_bits = (hist_bins - 1).bit_length()
key_bits = Uint64.width
radix_passes = (key_bits + radix_bits - 1) // radix_bits
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
hist_chunks = (hist_bins + tb_size - 1) // tb_size
warps_per_block = tb_size // cute.arch.WARP_SIZE
@dataclass(frozen=True)
class CompileKey:
topk: int
num_candidates: int
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
tb_size = StableTopKFromGatheredCandidatesKernel.tb_size
hist_bins = StableTopKFromGatheredCandidatesKernel.hist_bins
radix_bits = StableTopKFromGatheredCandidatesKernel.radix_bits
key_bits = StableTopKFromGatheredCandidatesKernel.key_bits
radix_passes = StableTopKFromGatheredCandidatesKernel.radix_passes
final_radix_bits = StableTopKFromGatheredCandidatesKernel.final_radix_bits
hist_chunks = StableTopKFromGatheredCandidatesKernel.hist_chunks
warps_per_block = StableTopKFromGatheredCandidatesKernel.warps_per_block
topk = compile_key.topk
assert hist_bins == 1 << radix_bits
assert compile_key.num_candidates % tb_size == 0, (
"StableTopKFromGatheredCandidatesKernel requires candidate count "
f"to be a multiple of {tb_size}, got {compile_key.num_candidates}"
)
keys_per_thread = compile_key.num_candidates // tb_size
@cute.struct
class SharedStorage:
hist: cute.struct.MemRange[Int32, hist_bins]
committed_count: cute.struct.MemRange[Int32, 1]
running_count: cute.struct.MemRange[Int32, 1]
threshold_bin: cute.struct.MemRange[Int32, 1]
threshold_found: cute.struct.MemRange[Int32, 1]
include_threshold_bin: cute.struct.MemRange[Int32, 1]
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
warp_totals: cute.struct.MemRange[Int32, warps_per_block]
shared_storage = SharedStorage
@cute.jit
def warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
offset = 1 << i
partial = cute.arch.shuffle_sync_up(
val, offset=offset, mask_and_clamp=0
)
if lane >= offset:
val += partial
return val
@cute.jit
def block_scan_inclusive_i32(
val: Int32,
lane: Int32,
warp_id: Int32,
warp_scratch: cute.Tensor,
warps_per_block: int,
) -> Int32:
prefix = warp_scan_inclusive_i32(val, lane)
if lane == Int32(cute.arch.WARP_SIZE - 1):
warp_scratch[0, warp_id] = prefix
cute.arch.sync_threads()
if warp_id == Int32(0):
warp_total = Int32(0)
if lane < Int32(warps_per_block):
warp_total = warp_scratch[0, lane]
warp_prefix = warp_scan_inclusive_i32(warp_total, lane)
if lane < Int32(warps_per_block):
warp_scratch[0, lane] = warp_prefix - warp_total
cute.arch.sync_threads()
return prefix + warp_scratch[0, warp_id]
@cute.jit
def stable_key(score: Float32, token_id: Int32) -> Uint64:
bits = recast_val(score, Uint32)
mask = Uint32(0x80000000)
if (bits & Uint32(0x80000000)) != Uint32(0):
mask = Uint32(0xFFFFFFFF)
score_key = Uint64(bits ^ mask) << Uint64(32)
id_key = Uint64(~Uint32(token_id))
key = score_key | id_key
if token_id < Int32(0):
key = Uint64(0)
return key
@cute.jit
def prefix_matches(
key: Uint64,
prefix: Uint64,
prefix_bits: Int32,
):
matches = prefix_bits == Int32(0)
if prefix_bits != Int32(0):
shift = Int32(key_bits) - prefix_bits
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
return matches
@cute.jit
def radix_pass(
keys: cute.Tensor,
output: cute.Tensor,
storage,
tid: Int32,
step: Int32,
bits: int,
is_final_pass: bool,
):
hist_smem = storage.hist.get_tensor(cute.make_layout((hist_bins,)))
committed_count_smem = storage.committed_count.data_ptr()
running_count_smem = storage.running_count.data_ptr()
threshold_bin_smem = storage.threshold_bin.data_ptr()
threshold_found_smem = storage.threshold_found.data_ptr()
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
warp_totals_smem = storage.warp_totals.get_tensor(
cute.make_layout((1, warps_per_block))
)
prefix_bits = step * Int32(radix_bits)
num_bins = 1 << bits
block_scan_iterations = (num_bins + tb_size - 1) // tb_size
shift = Int32(key_bits) - prefix_bits - Int32(bits)
bin_mask = Uint64(num_bins - 1)
prefix = prefix_smem.load()
for chunk in cutlass.range_constexpr(hist_chunks):
hist_smem[tid + Int32(chunk * tb_size)] = Int32(0)
if tid == Int32(0):
running_count_smem.store(committed_count_smem.load())
include_threshold_bin_smem.store(Int32(0))
threshold_found_smem.store(Int32(0))
cute.arch.sync_threads()
for key_idx in cutlass.range_constexpr(keys_per_thread):
key = keys[key_idx]
if prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
cute.arch.atomic_add(
hist_smem.iterator + bin_idx,
Int32(1),
sem="relaxed",
scope="cta",
)
cute.arch.sync_threads()
lane = cute.arch.lane_idx()
warp_id = cute.arch.warp_idx()
# Each iteration scans one tb_size-wide slice of bins, high to low.
iter = Int32(0)
threshold_found = threshold_found_smem.load()
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
bin_idx = Int32(num_bins - 1) - (iter * Int32(tb_size) + tid)
count = hist_smem[bin_idx]
chunk_inclusive = block_scan_inclusive_i32(
count,
lane,
warp_id,
warp_totals_smem,
warps_per_block,
)
running_count = running_count_smem.load()
prior_in_scan_slice = chunk_inclusive - count
remaining = Int32(topk) - running_count - prior_in_scan_slice
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
threshold_bin_smem.store(bin_idx)
if count <= remaining or cutlass.const_expr(is_final_pass):
include_threshold_bin_smem.store(Int32(1))
threshold_found_smem.store(Int32(1))
# Barrier: every thread must finish reading running_count for this
# slice before tb_size-1 advances it, else a warp racing ahead to
# the store makes a lagging thread double-count the slice total
# (-> remaining too small -> threshold too high -> under-fill).
cute.arch.sync_threads()
if tid == Int32(tb_size - 1):
running_count_smem.store(running_count + chunk_inclusive)
cute.arch.sync_threads()
threshold_found = threshold_found_smem.load()
iter += Int32(1)
threshold = threshold_bin_smem.load()
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
for key_idx in cutlass.range_constexpr(keys_per_thread):
key = keys[key_idx]
if prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
selected = bin_idx > threshold
if should_include_threshold:
selected = selected or bin_idx == threshold
if selected:
dst = cute.arch.atomic_add(
committed_count_smem,
Int32(1),
sem="relaxed",
scope="cta",
)
if dst < Int32(topk):
output[dst] = recast_val(~Uint32(key), Int32)
cute.arch.sync_threads()
pass_finished = include_threshold_bin_smem.load()
if tid == Int32(0) and pass_finished == Int32(0):
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
cute.arch.sync_threads()
return pass_finished
@cute.kernel
def device_kernel(input: cute.Tensor, out: cute.Tensor):
row, _, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
input_row = input[row, None, None]
output_row = out[row, None]
keys = cute.make_rmem_tensor((keys_per_thread,), Uint64)
smem = cutlass.utils.SmemAllocator()
storage = smem.allocate(shared_storage, 8)
committed_count_smem = storage.committed_count.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
for i in range(tid, topk, tb_size):
output_row[i] = Int32(-1)
for key_idx in cutlass.range_constexpr(keys_per_thread):
col = tid + Int32(key_idx * tb_size)
score = Float32(input_row[col, 0])
token_id = Int32(input_row[col, 1])
keys[key_idx] = stable_key(score, token_id)
if tid == Int32(0):
committed_count_smem.store(Int32(0))
prefix_smem.store(Uint64(0))
cute.arch.sync_threads()
step = Int32(0)
finished = Int32(0)
while finished == Int32(0) and step < Int32(radix_passes - 1):
finished = radix_pass(
keys,
output_row,
storage,
tid,
step,
radix_bits,
False,
)
step += Int32(1)
if finished == Int32(0):
radix_pass(
keys,
output_row,
storage,
tid,
Int32(radix_passes - 1),
final_radix_bits,
True,
)
@cute.jit
def host_entrypoint(
gathered: cute.Tensor,
out: cute.Tensor,
stream: CUstream,
):
grid = (gathered.shape[0], 1, 1)
device_kernel(gathered, out).launch(
grid=grid,
block=(tb_size, 1, 1),
stream=stream,
)
return host_entrypoint
def dispatch( # type: ignore[override]
self,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(**compile_key_fields)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
topk, _ = _dcp_merge_topk_and_interleave(vllm_config)
if topk <= 0:
return []
return self._trace_dispatch(self.dispatch)(
topk=topk,
num_candidates=topk * dcp_world_size,
)
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
num_rows = cute.sym_int()
gathered = cute.runtime.make_fake_tensor(
Float32,
(num_rows, compile_key.num_candidates, 2),
stride=(cute.sym_int64(divisibility=2), 2, 1),
assumed_align=8,
)
out = make_fake_tensor(
Int32,
(num_rows, compile_key.topk),
divisibility=1,
)
return gathered, out
@kernel_launcher
def __call__(
self,
gathered: torch.Tensor,
out: torch.Tensor,
*,
topk: int,
) -> CuTeDSLLaunchSpec[CompileKey]:
compile_key = self.dispatch(
topk=topk,
num_candidates=gathered.shape[1],
)
return compile_key, (gathered, out)
_PACK_DCP_TOPK_CANDIDATES_KERNEL = PackDCPTopkCandidatesKernel()
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL = StableTopKFromGatheredCandidatesKernel()
@@ -59,10 +59,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
TritonW4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
XPUW4A8IntLinearKernel,
XPUwNa16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
ZentorchWNA16LinearKernel,
)
@@ -91,9 +87,6 @@ from vllm.model_executor.kernels.linear.mxfp4.marlin import (
from vllm.model_executor.kernels.linear.mxfp4.pytorch import (
TorchMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.xpu import (
XPUMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp6 import (
MxFp6LinearKernel,
MxFp6LinearLayerConfig,
@@ -132,9 +125,6 @@ from vllm.model_executor.kernels.linear.mxfp8.pytorch import (
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.xpu import (
XPUMxFp8LinearKernel,
)
from vllm.model_executor.kernels.linear.nvfp4 import (
NvFp4LinearKernel,
NvFp4LinearLayerConfig,
@@ -228,11 +218,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFP8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
XPUFp8BlockScaledMMKernel,
XPUW8A8FP8LinearKernel,
XPUW8A16FP8LinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
ZentorchInt8ScaledMMLinearKernel,
)
@@ -373,13 +358,6 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
EmulationMxfp6LinearKernel,
EmulationMxfp4LinearKernel,
},
"xpu": {
XPUW8A8FP8LinearKernel,
XPUFp8BlockScaledMMKernel,
},
"xpu_woq": {
XPUW8A16FP8LinearKernel,
},
}
@@ -444,7 +422,6 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
HummingInt8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
PlatformEnum.XPU: [TritonInt8ScaledMMLinearKernel],
}
# in priority/performance order (when available)
@@ -473,12 +450,6 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
],
PlatformEnum.XPU: [
XPUW8A16FP8LinearKernel,
XPUW8A8FP8LinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
],
}
@@ -504,11 +475,6 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
CPUFp8W8A8BlockScaledMMKernel, # W8A8 preferred; falls back to W8A16 below
CPUFp8BlockScaledMMKernel,
],
PlatformEnum.XPU: [
XPUFp8BlockScaledMMKernel,
TritonFp8BlockScaledMMKernel,
BlockWiseTorchFP8ScaledMMLinearKernel,
],
}
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
@@ -522,9 +488,6 @@ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]
PlatformEnum.CPU: [
# To be added
],
PlatformEnum.XPU: [
XPUW8A16FP8LinearKernel,
],
}
# in priority/performance order (when available)
@@ -545,10 +508,6 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
ConchLinearKernel,
ExllamaLinearKernel,
],
PlatformEnum.XPU: [
XPUW4A8IntLinearKernel,
XPUwNa16LinearKernel,
],
PlatformEnum.CPU: [
Dynamic4bitLinearKernel,
ZentorchWNA16LinearKernel,
@@ -574,11 +533,6 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
RocmDotScaledMxfp8LinearKernel,
EmulationMxfp8LinearKernel,
],
PlatformEnum.XPU: [
XPUMxFp8LinearKernel,
TorchMxFp8LinearKernel,
EmulationMxfp8LinearKernel,
],
}
_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
@@ -626,10 +580,6 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
AiterMxfp4LinearKernel,
EmulationMxfp4LinearKernel,
],
PlatformEnum.XPU: [
XPUMxFp4LinearKernel,
TorchMxFp4LinearKernel,
],
}
# TODO make all kernels inherit from MMLinearKernel
@@ -1348,7 +1298,6 @@ __all__ = [
"MacheteLinearKernel",
"MarlinLinearKernel",
"TritonW4A16LinearKernel",
"XPUW4A8IntLinearKernel",
"XPUwNa16LinearKernel",
"init_mxfp8_linear_kernel",
"Mxfp8LinearKernel",
@@ -1373,7 +1322,6 @@ __all__ = [
"FlashInferCutlassMxfp8LinearKernel",
"FlashInferTrtllmMxfp8LinearKernel",
"MarlinMxfp8LinearKernel",
"XPUMxFp8LinearKernel",
"TorchMxFp8LinearKernel",
"EmulationMxfp8LinearKernel",
"CutlassNvFp4LinearKernel",
@@ -38,10 +38,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
TritonW4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
XPUW4A8IntLinearKernel,
XPUwNa16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
ZentorchWNA16LinearKernel,
)
@@ -60,7 +56,6 @@ __all__ = [
"RDNA3W4A16LinearKernel",
"RDNAHybridW4A16LinearKernel",
"TritonW4A16LinearKernel",
"XPUW4A8IntLinearKernel",
"XPUwNa16LinearKernel",
"ZentorchWNA16LinearKernel",
]
@@ -1,218 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
_XPUWNA16_SUPPORTED_QUANT_TYPES = (scalar_types.uint4, scalar_types.uint4b8)
logger = init_logger(__name__)
class XPUwNa16LinearKernel(MPLinearKernel):
@classmethod
def get_min_capability(cls) -> int:
return -1
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUwNa16 only supported on XPU"
if c.act_type != torch.bfloat16 and c.act_type != torch.float16:
return False, "XPUwNa16 only supports BF16/FP16 activations"
if c.weight_type not in _XPUWNA16_SUPPORTED_QUANT_TYPES:
return (
False,
f"Quant type ({c.weight_type}) not supported by "
"XPUwNa16, supported types are: "
f"{_XPUWNA16_SUPPORTED_QUANT_TYPES}",
)
if c.group_size != -1 and c.group_size % 32 != 0:
return (
False,
f"Group size ({c.group_size}) not supported by "
"XPUwNa16, supported group sizes are multiples of 32",
)
if c.partition_weight_shape[0] % 32 != 0:
return (
False,
f"Input size ({c.partition_weight_shape[0]}) not supported by "
"XPUwNa16, supported sizes are multiples of 32",
)
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module):
# Default name since marlin requires empty parameter for zp,
# TODO: remove this requirement from marlin (allow optional tensors)
if self.w_zp_name is None:
self.w_zp_name = "w_zp"
need_transpose = False
qweight_shape = getattr(layer, self.w_q_name).shape
scale_shape = getattr(layer, self.w_s_name).shape
# gptq marlin and compressed tensors wna16 expect different default
# layouts for weight and scale, so we check the shapes to determine
# if we need to transpose
if qweight_shape[0] != scale_shape[0]:
need_transpose = True
if need_transpose:
getattr(layer, self.w_q_name).data = (
getattr(layer, self.w_q_name).data.t().contiguous()
)
getattr(layer, self.w_s_name).data = getattr(layer, self.w_s_name).data
else:
getattr(layer, self.w_s_name).data = (
getattr(layer, self.w_s_name).data.t().contiguous()
)
if self.config.zero_points:
# (FIXME): maybe zero points should also be transposed.
getattr(layer, self.w_zp_name).data = (
getattr(layer, self.w_zp_name).data.t().contiguous()
)
else:
weight_zero_point = torch.Tensor([8]).to(torch.int8).to("xpu")
setattr(
layer, self.w_zp_name, Parameter(weight_zero_point, requires_grad=False)
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1])
w_q, w_s, w_zp = self._get_weight_params(layer)
out = torch.ops._xpu_C.int4_gemm_w4a16(
reshaped_x,
w_q.t(),
bias if bias is not None else None,
w_s,
w_zp,
self.config.group_size,
None, # Retained by the external XPU op ABI.
)
return out
class XPUW4A8IntLinearKernel(MPLinearKernel):
"""XPU kernel for W4A8 integer quantization using oneDNN int4_gemm_w4a8.
Weights are symmetric group-quantized int4 packed as uint4.
Activations are dynamically quantized per-token to symmetric int8.
"""
@classmethod
def get_min_capability(cls) -> int:
return -1
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUW4A8Int only supported on XPU"
if c.act_type not in (torch.bfloat16, torch.float16):
return False, "XPUW4A8Int requires BF16/FP16 activations"
if c.weight_type != scalar_types.int4:
return (
False,
f"XPUW4A8Int requires int4 weights, got {c.weight_type}",
)
if c.zero_points:
return False, "XPUW4A8Int only supports symmetric weight quantization"
if c.group_size != -1 and c.group_size % 32 != 0:
return (
False,
f"Group size ({c.group_size}) not supported by XPUW4A8Int, "
"must be a multiple of 32",
)
in_size, out_size = c.partition_weight_shape
if in_size % 8 != 0 or out_size % 8 != 0:
return (
False,
f"in/out sizes ({in_size}, {out_size}) must be multiples of 8",
)
if c.act_type != torch.float16:
logger.warning_once(
"XPUW4A8IntLinearKernel is running with model dtype %s, "
"but int4_gemm_w4a8 produces float16 output. Recommend "
"setting --dtype float16 for best performance.",
c.act_type,
)
return True, None
def _pack_int4_weight(self, w: torch.Tensor) -> torch.Tensor:
# w is [N, K] int8 with values in [-8, 7]
w_u4 = w.to(torch.int32) + 8 # shift to [0, 15]
w_u4 = w_u4.reshape(w.shape[0], w.shape[1] // 8, 8) # [N, K/8, 8]
shifts = torch.arange(0, 32, 4, dtype=torch.int32, device=w.device)
packed = ((w_u4 & 0xF) << shifts[None, None, :]).sum(dim=2).to(torch.int32)
return packed
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale.data = layer.weight_scale.data.t().contiguous()
device = layer.weight_packed.device
# TODO: support asymmetric quantization
weight_zero_point = torch.tensor([8], dtype=torch.int8, device=device)
layer.weight_zero_point = Parameter(weight_zero_point, requires_grad=False)
# weight_packed is [out, in] int8, signed int4 values in [-8, 7]
w = layer.weight_packed.data # [out, in]
# TODO: implement asym case
packed = self._pack_int4_weight(w) # [out, in/8] packed uint4
replace_parameter(
layer,
self.w_q_name,
torch.nn.Parameter(packed, requires_grad=False),
)
# Free the original unpacked int8 weight (still registered as "weight")
# to avoid double-storing both int8 [N, K] and int32 [N, K/8] in memory.
delattr(layer, "weight")
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1]) # [M, K]
from vllm._xpu_ops import xpu_ops as ops
# TODO: static and asymmetric quantization case
# Common code for CompressedTensorsW4A8Int does not read act symmetry data
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
reshaped_x, True, 8
)
out = torch.ops._xpu_C.int4_gemm_w4a8(
quant_x,
x_scale,
x_zero,
layer.weight_packed.t(),
layer.weight_scale,
layer.weight_zero_point,
self.config.group_size,
None, # Retained by the external XPU op ABI.
bias,
)
return out.to(x.dtype)
@@ -1,58 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
xpu_mxfp4_quantize as quant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
class XPUMxFp4LinearKernel(MxFp4LinearKernel):
"""MXFP4 W4A4 GEMM on XPU."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUMxFp4 only support on XPU"
return True, None
@classmethod
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key != kMxfp4Dynamic:
return False, "only supports MXFP4 dynamic activation"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight = layer.weight.view(torch.float4_e2m1fn_x2)
replace_parameter(layer, "weight", weight.data.t())
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
weight_scale = weight_scale.t().contiguous()
replace_parameter(layer, "weight_scale", weight_scale.data)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_dtype = x.dtype
x_fp4, x_blockscale = quant_mxfp4(x)
return torch.ops._xpu_C.fp4_gemm(
x_fp4,
layer.weight,
x_blockscale,
layer.weight_scale,
out_dtype,
bias,
)
@@ -1,82 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
xpu_mxfp8_quantize as quant_mxfp8,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
class XPUMxFp8LinearKernel(Mxfp8LinearKernel):
"""MXFP8 W8A8 GEMM on XPU."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUMxFp8 only support on XPU"
return True, None
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Checkpoint scale is [N, K//32] (one E8M0 scale per 32 elements).
# oneDNN fp8_gemm requires contiguous [K//32, N] layout. Store the
# transposed contiguous buffer as a .t() view so that:
# - dequantization consumers still see the checkpoint shape
# - apply_weights recovers the oneDNN layout via .t() at zero cost
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
scale_kn = weight_scale.data.t().contiguous()
replace_parameter(layer, "weight_scale", scale_kn.t())
if getattr(layer, "is_bmm", False):
self._prepare_bmm_params(layer, scale_kn)
def _prepare_bmm_params(
self, layer: torch.nn.Module, scale_kn: torch.Tensor
) -> None:
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
Splits scale [K//32, N_total] into [G, K//32, N_per_group] and weight
[N_total, K] into contiguous [G, K, N_per_group] for batch GEMM.
"""
batch = layer.bmm_batch_size
k_blocks, n_blocks = scale_kn.shape
layer.bmm_scale = (
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
.permute(1, 0, 2)
.contiguous()
)
w = layer.weight
n_total, k = w.shape
layer.bmm_weight = (
w.reshape(batch, n_total // batch, k).permute(0, 2, 1).contiguous()
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_dtype = x.dtype
x_fp8, x_scale = quant_mxfp8(x)
# Weight is [N, K]; .t() gives a [K, N] view without copying.
# Scale is stored as [N, K//32]; .t() recovers the contiguous
# [K//32, N] buffer that oneDNN expects.
return torch.ops._xpu_C.fp8_gemm(
x_fp8,
layer.weight.t(),
out_dtype,
x_scale,
layer.weight_scale.t(),
bias,
)
@@ -42,9 +42,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFP8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
XPUFp8BlockScaledMMKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
ZentorchInt8ScaledMMLinearKernel,
)
@@ -73,5 +70,4 @@ __all__ = [
"CPUFp8BlockScaledMMKernel",
"CPUFp8PerTensorScaledMMLinearKernel",
"CPUFP8W8A8ScaledMMLinearKernel",
"XPUFp8BlockScaledMMKernel",
]
@@ -1,284 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Sequence
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel
from .ScaledMMLinearKernel import FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig
class XPUW8A8FP8LinearKernel(FP8ScaledMMLinearKernel):
_SUPPORTED_ACT_QUANT_KEYS = {
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
}
_SUPPORTED_WEIGHT_QUANT_KEYS = {
kFp8StaticChannelSym,
kFp8StaticTensorSym,
}
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUW8A8FP8Linear only support on XPU"
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
if c.weight_quant_key not in cls._SUPPORTED_WEIGHT_QUANT_KEYS:
return (
False,
"XPUW8A8FP8Linear only support per-channel and per-tensor quantization",
)
if c.activation_quant_key not in cls._SUPPORTED_ACT_QUANT_KEYS:
return (
False,
"XPUW8A8FP8Linear only support per-tensor and per-token activation "
"quantization",
)
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
return False, "XPUW8A8FP8Linear only support FP8 weight dtype"
if c.activation_quant_key.dtype not in {
torch.float8_e5m2,
torch.float8_e4m3fn,
}:
return False, "XPUW8A8FP8Linear only support FP8 activation dtype"
return True, None
def __init__(
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
) -> None:
super().__init__(c, layer_param_names)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Ensure weight is stored as C-contiguous [K, N] (KN layout).
Checkpoints store weight as [N, K]; fp8_gemm requires [K, N],
C-contiguous. Three incoming layouts are possible:
• [N, K] C-contiguous ← direct checkpoint → .t().contiguous()
• [K, N] Fortran-order ← fp8.py's weight.t() → .contiguous()
• [K, N] C-contiguous ← already correct → no-op
For square weights (K == N) the shape is ambiguous; contiguity is used
as a proxy: C-contiguous ≡ checkpoint [N, K] (needs transpose);
Fortran-order ≡ fp8.py already transposed (needs only contiguous).
"""
K = getattr(layer, "input_size_per_partition", self.config.weight_shape[1])
N = getattr(layer, "output_size_per_partition", self.config.weight_shape[0])
w = layer.weight
if w.shape not in {(K, N), (N, K)}:
raise ValueError(
f"XPUFP8ScaledMM expects weight shape ({K},{N}) or ({N},{K}), "
f"but got {tuple(w.shape)}"
)
needs_transpose = w.shape == (N, K) if K != N else w.is_contiguous()
layer_weight = w.t() if needs_transpose else w
replace_parameter(layer, "weight", layer_weight)
ws = layer.weight_scale
if ws.numel() == 1:
replace_parameter(layer, "weight_scale", ws.reshape(1))
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
# B is C-contiguous [K, N] from process_weights_after_loading.
# fp8_gemm routes on scale dtype (float32) and numel:
# As [1] → per-tensor (numel==1 branch)
# As [M,1] → per-token (group={1,K} branch, broadcast across K)
# Bs [1] → per-tensor
# Bs [N] → per-channel (mask=bit1 branch)
# No shape manipulation needed here.
output = torch.ops._xpu_C.fp8_gemm(A, B, out_dtype, As, Bs, bias)
return output.view(*output_shape)
class XPUW8A16FP8LinearKernel(FP8ScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUW8A16FP8Linear only support on XPU"
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
if c.weight_quant_key not in {kFp8StaticChannelSym, kFp8StaticTensorSym}:
return (
False,
"XPUW8A16FP8Linear only support per-channel and per-tensor "
"quantization",
)
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
return False, "XPUW8A16FP8Linear only support FP8 weight dtype"
return True, None
def __init__(
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
) -> None:
assert self.can_implement(c)[0]
assert self.is_supported()[0]
self.config = c
self.layer_param_names = layer_param_names
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# fp8_gemm_w8a16 expects weight in [in, out] layout.
# Transpose if weight is still in [out, in] layout.
# For square matrices, use contiguity as tie-breaker:
# checkpoint weights are contiguous, .t() views are not.
weight = layer.weight
out_features, in_features = self.config.weight_shape
if weight.shape == (out_features, in_features) and (
in_features != out_features or weight.is_contiguous()
):
replace_parameter(layer, "weight", weight.data.t())
# else: already in [in, out] layout — no-op
weight_scale = layer.weight_scale.t().contiguous()
replace_parameter(layer, "weight_scale", weight_scale.data)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
weight = layer.weight
weight_scale = layer.weight_scale
return torch.ops._xpu_C.fp8_gemm_w8a16(x, weight, weight_scale, bias)
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
pass
class XPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUFp8BlockScaledMM only support on XPU"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module):
super().process_weights_after_loading(layer)
scale_attr = (
"weight_scale_inv" if hasattr(layer, "weight_scale_inv") else "weight_scale"
)
scale = getattr(layer, scale_attr)
# Ragged N (N % block_n != 0): oneDNN needs n_blocks to divide N.
# Weight untouched; only repeat scale rows to a finer N-group gn that
# divides both N and block_n (gn = gcd(N, block_n)):
# scale [ceil(N/block_n), K/block_k] --> [N/gn, K/block_k]
# oneDNN only accepts gn that is a multiple of 16, and gcd(N, block_n)
# is a power of two (block_n=128), so gn must be >= 16. No-op when
# N % block_n == 0.
block_n, block_k = self.weight_group_shape
N, K = layer.weight.shape
if N % block_n != 0:
gn = math.gcd(N, block_n)
assert gn % 16 == 0, (
f"XPU block-scaled FP8: N ({N}) yields group width {gn}, but "
f"oneDNN only supports multiples of 16; this weight shape is "
f"unsupported."
)
col_start = torch.arange(N // gn, device=scale.device) * gn
src_idx = torch.div(col_start, block_n, rounding_mode="floor")
scale = scale.index_select(0, src_idx).contiguous()
# Ragged K needs the runtime activation scale expanded too, which we
# don't handle; DeepSeek/GLM keep K block-aligned, so fail loudly.
assert K % block_k == 0, (
f"XPU block-scaled FP8 requires K ({K}) to be a multiple of the "
f"weight block size ({block_k}); ragged-K weights are unsupported."
)
# Checkpoint scale is [n_blocks, k_blocks] (one value per block tile).
# oneDNN fp8_gemm requires contiguous [k_blocks, n_blocks] layout.
# We store the transposed contiguous buffer as a .t() view so that:
# - MLA's scaled_dequantize still sees [n_blocks, k_blocks] shape
# - apply_block_scaled_mm recovers the contiguous buffer via .t()
scale_kn = scale.data.t().contiguous() # [k_blocks, n_blocks]
replace_parameter(layer, scale_attr, scale_kn.t()) # view: [n_blocks, k_blocks]
if getattr(layer, "is_bmm", False):
self._prepare_bmm_params(layer, scale_kn)
def _prepare_bmm_params(
self, layer: torch.nn.Module, scale_kn: torch.Tensor
) -> None:
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
Splits scale [k_blocks, n_blocks] into [G, k_blocks, n_blocks_per_group]
and weight [N_total, K] into [G, K, N_per_group] for batch GEMM.
"""
batch = layer.bmm_batch_size
k_blocks, n_blocks = scale_kn.shape
layer.bmm_scale = (
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
.permute(1, 0, 2)
.contiguous()
)
w = layer.weight
N_total, K = w.shape
layer.bmm_weight = w.reshape(batch, N_total // batch, K).permute(
0, 2, 1
) # [G, K, N_per_group]
def apply_block_scaled_mm(
self,
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
) -> torch.Tensor:
# B is [N, K]; .t() gives [K, N] view (no copy).
# Bs is stored as [n_blocks, k_blocks] view; .t() recovers the
# contiguous [k_blocks, n_blocks] buffer that oneDNN expects.
return torch.ops._xpu_C.fp8_gemm(
A,
B.t(),
self.config.out_dtype,
As,
Bs.t(),
torch.Tensor(),
)
@@ -1,36 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .aiter import *
from .cpu import *
from .tilelang import *
from .torch import *
from .triton import *
__all__ = [
"mhc_pre_cuda",
"mhc_post_cuda",
"mhc_fused_post_pre_cuda",
"hc_head_fused_kernel_cuda",
"mhc_pre_aiter",
"mhc_pre_delayed_aiter",
"mhc_fused_post_pre_delayed_rms_norm_aiter",
"mhc_post_aiter",
"mhc_fused_post_pre_aiter",
"hc_head_fused_aiter",
"mhc_pre_cpu",
"mhc_post_cpu",
"hc_head_fused_cpu",
"mhc_pre_tilelang",
"mhc_post_tilelang",
"mhc_fused_post_pre_tilelang",
"hc_head_fused_tilelang",
"mhc_pre_torch",
"mhc_post_torch",
"mhc_fused_post_pre_torch",
"hc_head_fused_torch",
"mhc_pre_mix_triton",
"mhc_pre_triton",
"mhc_post_triton",
"mhc_fused_post_pre_triton",
"hc_head_fused_triton",
]
-428
View File
@@ -1,428 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.utils.torch_utils import direct_register_custom_op
def mhc_pre_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
norm_weight: optional RMSNorm weight fused into the pre kernel
norm_eps: epsilon for the fused RMSNorm when norm_weight is set
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_pre(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_weight,
norm_eps,
)
def _mhc_pre_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
# Create empty tensors with correct shapes for meta device / shape inference
post_mix = torch.empty(
*outer_shape,
hc_mult,
1,
dtype=torch.float32,
device=residual.device,
)
comb_mix = torch.empty(
*outer_shape,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
layer_input = torch.empty(
*outer_shape,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
return post_mix, comb_mix, layer_input
def mhc_pre_delayed_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""MHC pre with the pre-mix carried in from the previous sublayer.
Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix*
rather than the gate computed here, and that gate is returned as the
pre-mix for the next sublayer seam.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
pre_mix: shape (..., hc_mult) from the previous sublayer, or None at
model entry to select residual stream zero.
sublayer_out: attention or FFN output, shape (..., hidden_size). When
given with the two mixes below, the preceding post block is
applied here so AITER can fold it into the pre projection.
post_layer_mix: shape (..., hc_mult, 1), post gate for that block.
comb_res_mix: shape (..., hc_mult, hc_mult), residual comb for it.
residual_out: shape (..., hc_mult, hidden_size), written with the post
block's new residual. Required exactly when the post is folded in.
Returns:
post_mix: shape (..., hc_mult, 1), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
next_pre_mix: shape (..., hc_mult), dtype torch.float32
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_pre_delayed(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
residual_out,
)
def _mhc_pre_delayed_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
return (
torch.empty(
*outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device
),
torch.empty(
*outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device
),
torch.empty(
*outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device
),
torch.empty(*outer_shape, hc_mult, dtype=torch.float32, device=residual.device),
)
def mhc_fused_post_pre_delayed_rms_norm_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None,
sublayer_out: torch.Tensor | None,
post_layer_mix: torch.Tensor | None,
comb_res_mix: torch.Tensor | None,
norm_weight: torch.Tensor,
norm_eps: float,
residual_out: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
""":func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in:
``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``."""
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
norm_weight,
norm_eps,
residual_out,
)
def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None,
sublayer_out: torch.Tensor | None,
post_layer_mix: torch.Tensor | None,
comb_res_mix: torch.Tensor | None,
norm_weight: torch.Tensor,
norm_eps: float,
residual_out: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
return _mhc_pre_delayed_aiter_fake(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
def mhc_post_aiter(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_post(
x,
residual,
post_layer_mix,
comb_res_mix,
)
def _mhc_post_aiter_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
return torch.empty_like(residual)
def mhc_fused_post_pre_aiter(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fused mHC post + next mHC pre on ROCm via AITER.
Returns residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur.
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_fused_post_pre(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_weight,
norm_eps,
)
def _mhc_fused_post_pre_aiter_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
post_mix = torch.empty(
*outer_shape,
hc_mult,
1,
dtype=torch.float32,
device=residual.device,
)
comb_mix = torch.empty(
*outer_shape,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
layer_input = torch.empty(
*outer_shape,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
return torch.empty_like(residual), post_mix, comb_mix, layer_input
direct_register_custom_op(
op_name="mhc_pre_aiter",
op_func=mhc_pre_aiter,
mutates_args=[],
fake_impl=_mhc_pre_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_pre_delayed_aiter",
op_func=mhc_pre_delayed_aiter,
mutates_args=["residual_out"],
fake_impl=_mhc_pre_delayed_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_fused_post_pre_delayed_rms_norm_aiter",
op_func=mhc_fused_post_pre_delayed_rms_norm_aiter,
mutates_args=["residual_out"],
fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_post_aiter",
op_func=mhc_post_aiter,
mutates_args=[],
fake_impl=_mhc_post_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_fused_post_pre_aiter",
op_func=mhc_fused_post_pre_aiter,
mutates_args=[],
fake_impl=_mhc_fused_post_pre_aiter_fake,
)
-96
View File
@@ -1,96 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm import _custom_ops as ops
def mhc_pre_cpu(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""CPU-ported mHC pre block (see `mhc_pre_torch` for the eager reference).
The ported kernel (`hc_pre_fused_cpu`) only exposes one merged `hc_eps`
(used for both `hc_pre_eps`/`hc_sinkhorn_eps`) and hardcodes the post-mix
multiplier to 2.0 -- true of every real call site in
`models/deepseek_v4/cpu/model.py`, so this is not a capability loss here.
"""
assert n_splits == 1, "mhc_pre_cpu does not support n_splits != 1"
assert hc_pre_eps == hc_sinkhorn_eps, (
"mhc_pre_cpu requires hc_pre_eps == hc_sinkhorn_eps (single merged hc_eps)"
)
assert hc_post_mult_value == 2.0, "mhc_pre_cpu hardcodes the post-mix multiplier"
hc_mult, hidden_size = residual.shape[-2:]
outer_shape = residual.shape[:-2]
x_flat = residual.reshape(-1, hc_mult, hidden_size)
layer_input, post, comb = ops.hc_pre_fused_cpu(
x_flat,
fn,
hc_scale,
hc_base,
hc_mult,
sinkhorn_repeat,
rms_eps,
hc_pre_eps,
)
post_mix = post.view(*outer_shape, hc_mult, 1)
comb_mix = comb.view(*outer_shape, hc_mult, hc_mult)
layer_input = layer_input.view(*outer_shape, hidden_size)
return post_mix, comb_mix, layer_input
def mhc_post_cpu(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
"""CPU-ported mHC post block (see `mhc_post_torch` for the eager reference)."""
hc_mult, hidden_size = residual.shape[-2:]
outer_shape = residual.shape[:-2]
x_flat = x.reshape(-1, hidden_size)
residual_flat = residual.reshape(-1, hc_mult, hidden_size)
post_flat = post_layer_mix.reshape(-1, hc_mult).float()
comb_flat = comb_res_mix.reshape(-1, hc_mult, hc_mult).float()
out = ops.hc_post_fused_cpu(x_flat, residual_flat, post_flat, comb_flat)
return out.view(*outer_shape, hc_mult, hidden_size)
def hc_head_fused_cpu(
hidden_states: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_norm_eps: float,
hc_eps: float,
) -> torch.Tensor:
"""CPU-ported HC head reduction (see `test_hc_head_cpu` in
tests/kernels/mhc/test_mhc_kernels.py for the eager reference this is tested
against).
The ported kernel's C++ signature takes `(hc_eps, norm_eps)` -- the
opposite order from this wrapper's `(rms_norm_eps, hc_eps)`, which matches
the real call site in `models/deepseek_v4/cpu/model.py`.
"""
hc_mult, hidden_size = hidden_states.shape[-2:]
outer_shape = hidden_states.shape[:-2]
hs_flat = hidden_states.reshape(-1, hc_mult, hidden_size)
out = ops.hc_head_fused_cpu(hs_flat, hc_fn, hc_scale, hc_base, hc_eps, rms_norm_eps)
return out.view(*outer_shape, hidden_size)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-147
View File
@@ -1,147 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
def mhc_pre_delayed_torch(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
x: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Reference for mHC pre using coefficients from the previous sublayer."""
hc_mult = residual.shape[1]
x = (residual.flatten(1) if x is None else x).float()
mixes = (x @ fn.t()) * torch.rsqrt(x.square().mean(-1, keepdim=True) + rms_eps)
pre = (
torch.sigmoid(mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]) + hc_pre_eps
)
post = (
torch.sigmoid(
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1]
+ hc_base[hc_mult : 2 * hc_mult]
)
* hc_post_mult_value
)
comb = mixes[:, 2 * hc_mult :].view(-1, hc_mult, hc_mult) * hc_scale[2]
comb = comb + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
comb = torch.softmax(comb, dim=-1) + hc_sinkhorn_eps
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
for _ in range(sinkhorn_repeat - 1):
comb = comb / (comb.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
layer_input = (
residual[:, 0]
if pre_mix is None
else (pre_mix.unsqueeze(-1) * residual.float()).sum(dim=1).to(residual.dtype)
)
return post.unsqueeze(-1), comb, layer_input, pre
def mhc_pre_torch(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
# Validate shapes
assert residual.dtype == torch.bfloat16
assert fn.dtype == torch.float32
assert hc_scale.dtype == torch.float32
assert hc_base.dtype == torch.float32
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
hc_mult2 = hc_mult * hc_mult
hc_mult3 = hc_mult * 2 + hc_mult2
hc_hidden_size = hc_mult * hidden_size
assert fn.shape[0] == hc_mult3
assert fn.shape[1] == hc_hidden_size
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
outer_shape = residual.shape[:-2]
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
fn_flat = fn
x = residual_flat.view(num_tokens, hc_mult * hidden_size).to(torch.float32)
mixes = torch.matmul(x, fn_flat.t())
sqrsum = x.square().sum(dim=-1, keepdim=True)
mixes = mixes * torch.rsqrt(sqrsum / (hc_mult * hidden_size) + rms_eps)
pre_logits = mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]
pre_mix = torch.sigmoid(pre_logits) + hc_pre_eps
post_logits = (
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1] + hc_base[hc_mult : 2 * hc_mult]
)
post_mix = torch.sigmoid(post_logits) * hc_post_mult_value
comb_logits = mixes[:, 2 * hc_mult :].view(num_tokens, hc_mult, hc_mult) * hc_scale[
2
] + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
comb_mix = torch.softmax(comb_logits, dim=-1) + hc_sinkhorn_eps
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
for _ in range(sinkhorn_repeat - 1):
comb_mix = comb_mix / (comb_mix.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
layer_input = torch.sum(
pre_mix.unsqueeze(-1) * residual_flat.to(torch.float32), dim=1
).to(torch.bfloat16)
return (
post_mix.view(*outer_shape, hc_mult, 1),
comb_mix.view(*outer_shape, hc_mult, hc_mult),
layer_input.view(*outer_shape, hidden_size),
)
def mhc_post_torch(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
mixed_residual = torch.einsum(
"...ij,...ih->...jh",
comb_res_mix.to(torch.float32),
residual.to(torch.float32),
)
post_term = post_layer_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)
return (mixed_residual + post_term).to(residual.dtype)
-349
View File
@@ -1,349 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn.functional as F
from torch import Tensor
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
@triton.jit
def _rmsnorm_nw_kernel(
x_ptr,
out_ptr,
stride_row,
D,
eps,
RBLOCK: tl.constexpr,
):
"""Weight-free RMSNorm Triton kernel: out = x * rsqrt(mean(x², -1) + eps)."""
row = tl.program_id(0)
cols = tl.arange(0, RBLOCK)
mask = cols < D
x = tl.load(
x_ptr + row * stride_row + cols,
mask=mask,
other=0.0,
eviction_policy="evict_first",
).to(tl.float32)
var = tl.sum(x * x, 0) / D
rstd = tl.rsqrt(var + eps)
out = (x * rstd).to(out_ptr.dtype.element_ty)
tl.store(out_ptr + row * D + cols, out, mask=mask, eviction_policy="evict_first")
def rmsnorm_nw(x: Tensor, eps: float) -> Tensor:
"""Weight-free RMSNorm over the last dimension.
Treats *x* as ``[num_rows, D]`` where ``num_rows = product(shape[:-1])``.
Returns a contiguous tensor with the same shape and dtype as *x*.
"""
orig_shape = x.shape
D = orig_shape[-1]
x_2d = x.reshape(-1, D)
num_rows = x_2d.shape[0]
out = torch.empty_like(x_2d)
RBLOCK = triton.next_power_of_2(D)
_rmsnorm_nw_kernel[(num_rows,)](
x_2d,
out,
x_2d.stride(0),
D,
eps,
RBLOCK=RBLOCK,
num_warps=1 if RBLOCK <= 512 else (4 if RBLOCK <= 4096 else 8),
)
return out.view(orig_shape)
@triton.jit
def _hc_head_reduce_store_kernel(
pre_ptr,
x_ptr,
out_ptr,
hidden_size: tl.constexpr,
hc_mult: tl.constexpr,
pre_stride_t: tl.constexpr,
pre_stride_m: tl.constexpr,
x_stride_t: tl.constexpr,
x_stride_m: tl.constexpr,
x_stride_h: tl.constexpr,
out_stride_t: tl.constexpr,
out_stride_h: tl.constexpr,
BLOCK_H: tl.constexpr,
):
token_idx = tl.program_id(0).to(tl.int64)
block_idx = tl.program_id(1)
offsets = block_idx * BLOCK_H + tl.arange(0, BLOCK_H)
mask = offsets < hidden_size
acc = tl.zeros((BLOCK_H,), dtype=tl.float32)
for mix_idx in tl.static_range(0, hc_mult):
pre = tl.load(pre_ptr + token_idx * pre_stride_t + mix_idx * pre_stride_m).to(
tl.float32
)
x = tl.load(
x_ptr
+ token_idx * x_stride_t
+ mix_idx * x_stride_m
+ offsets * x_stride_h,
mask=mask,
other=0.0,
).to(tl.float32)
acc += pre * x
tl.store(
out_ptr + token_idx * out_stride_t + offsets * out_stride_h,
acc,
mask=mask,
)
def hc_collapse_triton(x: Tensor, pre_mix: Tensor) -> Tensor:
"""Collapse BF16 residual streams with FP32 pre-mix coefficients."""
assert x.ndim == 3 and x.dtype == torch.bfloat16
num_tokens, hc_mult, hidden_size = x.shape
assert pre_mix.shape == (num_tokens, hc_mult)
assert pre_mix.dtype == torch.float32
out = torch.empty(num_tokens, hidden_size, dtype=x.dtype, device=x.device)
if num_tokens == 0:
return out
block_h = 1024
_hc_head_reduce_store_kernel[(num_tokens, triton.cdiv(hidden_size, block_h))](
pre_mix,
x,
out,
hidden_size,
hc_mult,
pre_mix.stride(0),
pre_mix.stride(1),
x.stride(0),
x.stride(1),
x.stride(2),
out.stride(0),
out.stride(1),
BLOCK_H=block_h,
num_warps=4,
# Preserve the separate FP32 multiply and sum in the Torch reference.
enable_fp_fusion=False,
)
return out
def _hc_collapse_triton_fake(x: Tensor, pre_mix: Tensor) -> Tensor:
return torch.empty(x.shape[0], x.shape[2], dtype=x.dtype, device=x.device)
@triton.jit
def _mhc_pre_mix_kernel(
gemm_ptr,
sqrsum_ptr,
hc_scale_ptr,
hc_base_ptr,
out_ptr,
splitk,
hc_mult: tl.constexpr,
gemm_stride_k,
gemm_stride_t,
gemm_stride_j,
sqrsum_stride_k,
sqrsum_stride_t,
out_stride_t,
out_stride_j,
inv_hc_hidden,
rms_eps,
hc_pre_eps,
SPLITK_BLOCK: tl.constexpr,
HC_BLOCK: tl.constexpr,
):
"""Recover the pre-mix gate from a split-k mHC pre GEMM output."""
token_idx = tl.program_id(0).to(tl.int64)
ks = tl.arange(0, SPLITK_BLOCK)
js = tl.arange(0, HC_BLOCK)
kmask = ks < splitk
jmask = js < hc_mult
# Only the first hc_mult GEMM columns feed the pre gate.
gemm = tl.load(
gemm_ptr
+ ks[:, None] * gemm_stride_k
+ token_idx * gemm_stride_t
+ js[None, :] * gemm_stride_j,
mask=kmask[:, None] & jmask[None, :],
other=0.0,
).to(tl.float32)
mixes = tl.sum(gemm, 0)
sqrsum = tl.load(
sqrsum_ptr + ks * sqrsum_stride_k + token_idx * sqrsum_stride_t,
mask=kmask,
other=0.0,
).to(tl.float32)
rstd = tl.rsqrt(tl.sum(sqrsum, 0) * inv_hc_hidden + rms_eps)
scale = tl.load(hc_scale_ptr).to(tl.float32)
base = tl.load(hc_base_ptr + js, mask=jmask, other=0.0).to(tl.float32)
pre = tl.sigmoid(mixes * rstd * scale + base) + hc_pre_eps
tl.store(out_ptr + token_idx * out_stride_t + js * out_stride_j, pre, mask=jmask)
def mhc_pre_mix_triton(
gemm_out: Tensor,
sqrsum: Tensor,
hc_scale: Tensor,
hc_base: Tensor,
hc_mult: int,
hc_hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
) -> Tensor:
"""Pre-mix gate for the delayed mHC pre, from AITER's split-k GEMM output.
AITER's ``mhc_pre_big_fuse`` consumes the unreduced ``[splitk, tokens,
hc_mult3]`` GEMM output and the matching row square-sums, but only returns
the post and comb gates. The delayed formulation also needs the pre gate,
to carry into the next sublayer seam. It is the same slice of the same
numbers, so recover it here rather than repeating the projection.
"""
assert gemm_out.ndim == 3 and gemm_out.dtype == torch.float32
assert sqrsum.ndim == 2 and sqrsum.dtype == torch.float32
splitk, num_tokens = gemm_out.shape[0], gemm_out.shape[1]
assert sqrsum.shape == (splitk, num_tokens)
out = torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=gemm_out.device)
if num_tokens == 0:
return out
_mhc_pre_mix_kernel[(num_tokens,)](
gemm_out,
sqrsum,
hc_scale,
hc_base,
out,
splitk,
hc_mult,
gemm_out.stride(0),
gemm_out.stride(1),
gemm_out.stride(2),
sqrsum.stride(0),
sqrsum.stride(1),
out.stride(0),
out.stride(1),
1.0 / hc_hidden_size,
rms_eps,
hc_pre_eps,
SPLITK_BLOCK=triton.next_power_of_2(splitk),
HC_BLOCK=triton.next_power_of_2(hc_mult),
num_warps=1,
# Match the separate FP32 multiply and add in the Torch reference.
enable_fp_fusion=False,
)
return out
def _mhc_pre_mix_triton_fake(
gemm_out: Tensor,
sqrsum: Tensor,
hc_scale: Tensor,
hc_base: Tensor,
hc_mult: int,
hc_hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
) -> Tensor:
return torch.empty(
gemm_out.shape[1], hc_mult, dtype=torch.float32, device=gemm_out.device
)
def hc_head_reduce_triton_kernel(
x: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
out: torch.Tensor,
norm_eps: float,
hc_eps: float,
) -> None:
x_flat = x.flatten(-2)
x_normed = rmsnorm_nw(x_flat, norm_eps)
mixes = F.linear(x_normed.float(), hc_fn)
pre = torch.sigmoid(mixes * hc_scale + hc_base) + hc_eps
hidden_size = x.shape[-1]
hc_mult = x.shape[-2]
block_h = 1024
_hc_head_reduce_store_kernel[(x.shape[0], (hidden_size + block_h - 1) // block_h)](
pre,
x,
out,
hidden_size,
hc_mult,
pre.stride(0),
pre.stride(1),
x.stride(0),
x.stride(1),
x.stride(2),
out.stride(0),
out.stride(1),
BLOCK_H=block_h,
num_warps=4,
)
def _hc_head_triton(
hs_flat: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
out: torch.Tensor,
hidden_size: int,
rms_eps: float,
hc_eps: float,
hc_mult: int,
) -> None:
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
if hs_flat.shape[0] == 0:
return
hc_head_reduce_triton_kernel(
hs_flat,
fn,
hc_scale,
hc_base,
out,
rms_eps,
hc_eps,
)
return
direct_register_custom_op(
op_name="hc_head_triton",
op_func=_hc_head_triton,
mutates_args=["out"],
)
direct_register_custom_op(
op_name="hc_collapse_triton",
op_func=hc_collapse_triton,
mutates_args=[],
fake_impl=_hc_collapse_triton_fake,
)
direct_register_custom_op(
op_name="mhc_pre_mix_triton",
op_func=mhc_pre_mix_triton,
mutates_args=[],
fake_impl=_mhc_pre_mix_triton_fake,
)
-101
View File
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from dataclasses import asdict, dataclass
from typing import Any
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
compute_num_split,
mhc_pre_big_fuse_with_norm_tilelang,
)
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.math_utils import cdiv
def compute_mhc_pre_num_splits(input_size: int, num_tokens: int) -> int:
splits = compute_num_split(64, input_size, cdiv(num_tokens, 64))
# Bound both GEMM and fused-normalization specializations during startup.
return 1 if splits == 1 else 4 if splits <= 4 else 16
class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
rms_eps: float
hc_pre_eps: float
hc_sinkhorn_eps: float
hc_post_mult_value: float
sinkhorn_repeat: int
norm_eps: float
n_splits: int
hc_mult: int
use_pre_mix_in: bool
rms_numel: int
save_pre_mix: bool = True
split_mode: str = "fused"
write_aux: bool = False
kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang)
def dispatch(self, *, n_splits, **fields) -> CompileKey: # type: ignore[override]
return self.CompileKey(n_splits=n_splits, **fields)
def get_warmup_keys(
self,
*,
max_tokens: int,
extra_splits: Iterable[int] = (),
**fields,
) -> list[CompileKey]:
# The split heuristic changes only at 64-token boundaries. Callers that
# reach this kernel through another GEMM (the fused post + pre-norm
# GEMM) pass the splits that GEMM picks in extra_splits.
splits = sorted(
set(extra_splits)
| (
{
compute_mhc_pre_num_splits(fields["rms_numel"], num_tokens)
for num_tokens in range(1, max_tokens + 1, 64)
}
if is_deep_gemm_supported()
else {1}
)
)
return self._trace_dispatch(self.dispatch)(n_splits=splits, **fields)
def compile(self, compile_key: CompileKey) -> None:
if compile_key not in self._compiled_cache:
self._compiled_cache[compile_key] = self.kernel.compile(
**asdict(compile_key)
)
def compile_many(self, compile_keys: Iterable[CompileKey]) -> None:
missing = list(
dict.fromkeys(
key for key in compile_keys if key not in self._compiled_cache
)
)
if len(missing) < 2:
return super().compile_many(missing)
# TileLang elaborates serially before compiling independent TIR modules.
compiled = self.kernel.par_compile(
[asdict(key) for key in missing], num_workers=min(4, len(missing))
)
self._compiled_cache.update(zip(missing, compiled, strict=True))
def __call__(self, *tensors, **fields):
compile_key = self.dispatch(
n_splits=tensors[0].shape[0],
**fields,
)
if not current_platform.is_cuda():
return self.kernel(*tensors, **asdict(compile_key))
return self._get_or_compile(compile_key)(*tensors)
MHC_PRE_NORM_KERNEL = MHCPreNormKernel()
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import cutlass
import cutlass.cute as cute
from vllm.vllm_flash_attn.cute import utils
@cute.jit
def dense_mask_mod(
batch: cute.TensorSSA,
head: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors: list,
) -> cute.TensorSSA:
dense_mask = aux_tensors[0]
batch_idx = utils.ssa_to_scalar(batch)
q_idx = utils.ssa_to_scalar(q_idx)
kv_idx = utils.ssa_to_scalar(kv_idx)
batch_stride, query_stride, _ = dense_mask.stride
aligned_mask = cute.make_tensor(
dense_mask.iterator,
cute.make_layout(
dense_mask.shape,
stride=(
cute.assume(batch_stride, divby=4),
cute.assume(query_stride, divby=4),
1,
),
),
)
mask_row = aligned_mask[batch_idx, q_idx, None]
mask_chunks = cute.flat_divide(mask_row, (4,))
mask_chunk = mask_chunks[None, (kv_idx >> 5) >> 2]
loaded = cute.make_rmem_tensor_like(mask_chunk)
cute.autovec_copy(mask_chunk, loaded)
return cute.recast_tensor(loaded, cutlass.Uint32).load()
dense_mask_mod.__vec_size__ = 128
@cute.jit
def offset_dense_mask_mod(
batch: cute.TensorSSA,
head: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors: list,
) -> cute.TensorSSA:
dense_mask = aux_tensors[0]
batch_idx = utils.ssa_to_scalar(batch)
q_idx = utils.ssa_to_scalar(q_idx)
key_start = dense_mask[batch_idx, 0, dense_mask.shape[2] - 1]
kv_idx = utils.ssa_to_scalar(kv_idx) + key_start
word_idx = kv_idx >> 5
bit_idx = cutlass.Uint32(kv_idx & 31)
word = dense_mask[batch_idx, q_idx, word_idx]
result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32)
result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx)
return result.load()
offset_dense_mask_mod.__vec_size__ = 32
@@ -119,11 +119,6 @@ if HAS_TRITON:
TritonExperts,
TritonWNA16Experts,
)
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExperts,
XPUExpertsFp8,
XPUExpertsMxFp4,
)
from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_experts,
get_config_file_name,
File diff suppressed because it is too large Load Diff
@@ -1,353 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kInt4Static,
kInt4Static32,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp8Dynamic,
kMxfp8Static,
)
from vllm.platforms import current_platform
if current_platform.is_xpu():
from vllm_xpu_kernels.fused_moe_interface import XpuFusedMoe
def prepare_fp8_moe_layer_for_xpu(
w13: torch.Tensor,
w13_scale: torch.Tensor,
w2: torch.Tensor,
w2_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
if w13_scale is not None and w13_scale.ndim == 3:
w13_scale = w13_scale.transpose(-1, -2).contiguous()
if w2_scale is not None and w2_scale.ndim == 3:
w2_scale = w2_scale.transpose(-1, -2).contiguous()
return (
w13.transpose(-1, -2).contiguous(),
w13_scale,
w2.transpose(-1, -2).contiguous(),
w2_scale,
)
class XPUExperts(mk.FusedMoEExpertsModular):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
self.fused_moe_impl: XpuFusedMoe | None = None
is_xe2_or_xe3 = torch.ops._xpu_C.is_xe2_arch() or torch.ops._xpu_C.is_xe3_arch()
if not is_xe2_or_xe3:
raise NotImplementedError(
"XPUExperts is only supported on Intel Xe2/Xe3 GPUs"
)
self._expects_unquantized_inputs = is_xe2_or_xe3
@property
def expects_unquantized_inputs(self) -> bool:
return self._expects_unquantized_inputs
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_xpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return True
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.RELU2_NO_MUL,
]
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(None, None),
(kFp8StaticTensorSym, None),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
workspace1 = (0,)
workspace2 = (0,)
output = (M, K)
return (workspace1, workspace2, output)
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
if self.fused_moe_impl is None:
topk = topk_ids.size(-1)
if (
self.quant_config is not None
and self.quant_config.weight_quant_dtype == "mxfp4"
):
w1 = w1.view(torch.float4_e2m1fn_x2)
w2 = w2.view(torch.float4_e2m1fn_x2)
self.fused_moe_impl = XpuFusedMoe(
w13=w1,
w13_scales=self.w1_scale,
w13_bias=self.w1_bias,
w2=w2,
w2_scales=self.w2_scale,
w2_bias=self.w2_bias,
n_experts_per_token=topk,
activation=activation.value,
num_experts=self.moe_config.num_local_experts,
ep_rank=self.moe_config.ep_rank,
ep_size=self.moe_config.ep_size,
expert_map=expert_map,
gemm1_clamp_limit=self.gemm1_clamp_limit,
)
assert self.fused_moe_impl is not None
if self.moe_config.ep_size > 1:
# Avoid topk_ids = -1, remap them to an
# expert not owned by this rank so they are skipped.
sentinel = global_num_experts - 1 if self.moe_config.ep_rank == 0 else 0
topk_ids = torch.where(topk_ids < 0, sentinel, topk_ids)
self.fused_moe_impl.apply(
output=output,
hidden_states=hidden_states,
topk_weights=topk_weights,
topk_ids=topk_ids,
a1q_scale=a1q_scale,
)
class XPUExpertsFp8(XPUExperts):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kFp8StaticTensorSym, None),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
class XPUExpertsMxFp8(XPUExpertsFp8):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
assert quant_config.quant_dtype == "mxfp8"
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kMxfp8Static, None),
(kMxfp8Static, kMxfp8Dynamic),
]
return (weight_key, activation_key) in SUPPORTED_W_A
class XPUExpertsBlockFp8(XPUExperts):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
class XPUExpertsWNA16(XPUExperts):
"""W4A16 INT4-symmetric MoE backed by `xpu_fused_moe(is_int4=True)`.
Weight layout when `is_int4=True` (per `xpu_fused_moe` docstring):
w13: [num_experts, 2*inter_size, hidden_size] contiguous int4-packed
w13_scales: [num_experts, 2*inter_size, hidden_size // group_size]
w2: [num_experts, hidden_size, inter_size] contiguous int4-packed
w2_scales: [num_experts, hidden_size, inter_size // group_size]
Pairs with `INCXPULinearMethod` for the linear layers; together they
cover full-attn + MoE on Intel XPU end-to-end without IPEX.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) in (
(kInt4Static, None),
(kInt4Static32, None),
)
class XPUExpertsMxFp4(XPUExperts):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# K = a1q.size(-1). When activations are pre-quantized packed mxfp4,
# K is the packed hidden_size (= logical / 2); the kernel output is at
# logical hidden_size (2 * K). When unquantized (bf16), K is already
# the logical size.
logical_K = K if self.expects_unquantized_inputs else 2 * K
return (0,), (0,), (M, logical_K)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kMxfp4Static, None),
(kMxfp4Static, kMxfp4Dynamic),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@@ -241,21 +241,8 @@ def backend_to_kernel_cls(
return [CutlassExpertsMxfp4]
elif backend == Mxfp4MoeBackend.TRITON:
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
OAITritonExperts,
OAITritonMxfp4ExpertsMonolithic,
)
# NOTE: prefer Monolithic > Modular, so return Monolithic first.
return [OAITritonMxfp4ExpertsMonolithic, OAITritonExperts]
elif backend == Mxfp4MoeBackend.TRITON_UNFUSED:
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
UnfusedOAITritonExperts,
)
return [UnfusedOAITritonExperts]
elif backend in (Mxfp4MoeBackend.TRITON, Mxfp4MoeBackend.TRITON_UNFUSED):
raise ImportError("gpt_oss triton_kernels experts were removed in this build.")
elif backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
@@ -1,213 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Producer side of the QuantizedActivation contract for activation layers.
Given an activation module and the downstream linear it feeds, fuse the
activation with that linear's input quantization into a single kernel when the
linear advertises a consumable input quantization key (see quant_activation.py).
Falls back to the plain activation when nothing matches, so a model forward can
always call maybe_fused_act_quant unconditionally.
This is the manual-fusion counterpart to ActivationQuantFusionPass: when fusion
fires here, the activation and quantization are already consumed, so a compiler
pass cannot fuse the same boundary again.
"""
from collections.abc import Callable
import torch
from vllm.model_executor.layers.activation import ReLUSquaredActivation, SiluAndMul
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.fusion.relu2_fp8_quant import (
relu_squared_static_fp8_quant,
)
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
def _silu_and_mul_fp8_static(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""SiluAndMul + FP8 static per-tensor quantization."""
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
result = torch.empty(out_shape, dtype=FP8_DTYPE, device=x.device)
# TODO(mgoin): read the consumer scale via the contract instead of reaching
# into the kernel-specific input_scale attribute.
scale = linear.input_scale
torch.ops._C.silu_and_mul_quant(result, x, scale)
return QuantizedActivation(
data=result,
scale=scale,
orig_dtype=x.dtype,
orig_shape=out_shape,
quant_key=kFp8StaticTensorSym,
)
def _silu_and_mul_fp8_dynamic_block(
x: torch.Tensor, linear: LinearBase, group_size: int, quant_key: QuantKey
) -> QuantizedActivation:
"""SiluAndMul + FP8 dynamic per-block quantization."""
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
x = x.reshape(-1, x.shape[-1])
num_tokens = x.shape[0]
num_groups = d // group_size
result = torch.empty((num_tokens, d), dtype=FP8_DTYPE, device=x.device)
scales = torch.empty((num_tokens, num_groups), dtype=torch.float32, device=x.device)
torch.ops._C.silu_and_mul_per_block_quant(
out=result,
input=x,
scales=scales,
group_size=group_size,
scale_ub=None,
is_scale_transposed=False,
)
return QuantizedActivation(
data=result.view(out_shape),
scale=scales.view(out_shape[:-1] + (num_groups,)),
orig_dtype=x.dtype,
orig_shape=out_shape,
quant_key=quant_key,
)
def _silu_and_mul_fp8_dynamic_128(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""SiluAndMul + FP8 dynamic per-group (group=128) quantization."""
return _silu_and_mul_fp8_dynamic_block(x, linear, 128, kFp8Dynamic128Sym)
def _silu_and_mul_nvfp4_dynamic(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""SiluAndMul + NVFP4 dynamic quantization."""
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
x = x.reshape(-1, x.shape[-1])
num_tokens = x.shape[0]
# NVFP4 packs 2 values into 1 byte
result = torch.empty((num_tokens, d // 2), dtype=FP4_DTYPE, device=x.device)
# Block scale output: swizzled tensor-core layout
# [num_m_tiles, num_k_tiles, 32, 4, 4] of int32-packed FP8 scales, so the
# row/col extents must be padded to the 128x4 tile like
# scaled_fp4_quant's allocator (create_fp4_scale_tensor). Each group of 16
# elements shares one FP8 scale.
rounded_m = round_up(num_tokens, 128)
rounded_n = round_up(d // 16, 4)
block_scale = torch.empty(
(rounded_m, rounded_n // 4), dtype=torch.int32, device=x.device
).view(FP8_DTYPE)
# The kernel folds the global scale into the block scales, and the
# consumer GEMM's alpha (= input_global_scale * weight_global_scale)
# divides it back out, so quantize with the reciprocal like the unfused
# scaled_fp4_quant path does.
input_global_scale_inv = getattr(linear, "input_global_scale_inv", None)
assert input_global_scale_inv is not None, (
"input_global_scale_inv is required for NVFP4 quantization"
)
torch.ops._C.silu_and_mul_nvfp4_quant(
result, block_scale, x, input_global_scale_inv
)
return QuantizedActivation(
data=result.view(out_shape[:-1] + (d // 2,)),
scale=block_scale,
orig_dtype=x.dtype,
orig_shape=out_shape,
quant_key=kNvfp4Dynamic,
)
# (activation module type, consumer input quantization key) -> fused producer.
# Add a row for each supported manual producer.
_FUSED_ACT_QUANT: dict[tuple[type, QuantKey], Callable] = {
(SiluAndMul, kFp8StaticTensorSym): _silu_and_mul_fp8_static,
}
def _relu_squared_static_fp8_quant_supported(
act_fn: torch.nn.Module,
x: torch.Tensor,
linear: LinearBase,
) -> bool:
"""Return whether the ReLU2 static-FP8 producer can consume this input."""
scale = getattr(linear, "input_scale", None)
return (
isinstance(act_fn, ReLUSquaredActivation)
and x.is_cuda
and x.dtype == torch.bfloat16
and x.is_contiguous()
and isinstance(scale, torch.Tensor)
and scale.dtype == torch.float32
and scale.device == x.device
and scale.numel() == 1
)
# Optional per-entry predicates preserve the unconditional fallback contract
# for producers whose supported inputs are narrower than their registry key.
_FUSED_ACT_QUANT_SUPPORT: dict[
tuple[type, QuantKey], Callable[[torch.nn.Module, torch.Tensor, LinearBase], bool]
] = {}
# This path is validated on SM90 and newer CUDA devices.
if current_platform.is_cuda() and current_platform.has_device_capability(90):
_RELU2_STATIC_FP8_KEY = (ReLUSquaredActivation, kFp8StaticTensorSym)
_FUSED_ACT_QUANT[_RELU2_STATIC_FP8_KEY] = relu_squared_static_fp8_quant
_FUSED_ACT_QUANT_SUPPORT[_RELU2_STATIC_FP8_KEY] = (
_relu_squared_static_fp8_quant_supported
)
# Add CUDA-specific entries for dynamic block quantization
if current_platform.is_cuda_alike():
_FUSED_ACT_QUANT[(SiluAndMul, kFp8Dynamic128Sym)] = _silu_and_mul_fp8_dynamic_128
# Add NVFP4 if supported (requires SM100+)
if current_platform.is_cuda() and hasattr(torch.ops._C, "silu_and_mul_nvfp4_quant"):
_FUSED_ACT_QUANT[(SiluAndMul, kNvfp4Dynamic)] = _silu_and_mul_nvfp4_dynamic
def maybe_fused_act_quant(
act_fn: torch.nn.Module,
x: torch.Tensor,
linear: LinearBase,
) -> "torch.Tensor | QuantizedActivation":
"""Apply act_fn, fusing the downstream linear's input quant when possible.
Returns a QuantizedActivation when a fused kernel matches the activation and
the consumer's effective input quantization key, else the plain activation.
"""
key = get_input_quant_key(linear)
if key is not None:
registry_key = (type(act_fn), key)
producer = _FUSED_ACT_QUANT.get(registry_key)
support = _FUSED_ACT_QUANT_SUPPORT.get(registry_key)
if producer is not None and (support is None or support(act_fn, x, linear)):
return producer(x, linear)
return act_fn(x)
@@ -1,14 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
)
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
return 16 if backend == "flashkda" else None
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
"""Store FlashKDA recurrent and convolution checkpoint states."""
@@ -1,704 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import torch
import torch.nn.functional as F
from transformers.configuration_utils import PreTrainedConfig
from vllm.config import (
VllmConfig,
get_current_vllm_config,
)
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import (
MiniMaxText01LinearAttention,
MiniMaxText01LinearKernel,
linear_attention_decode,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
RMSNormGated,
layernorm_fn,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
def _build_rope_parameters(config: PreTrainedConfig) -> dict | None:
rope_parameters = copy.deepcopy(getattr(config, "rope_parameters", None)) or {}
if "rope_theta" not in rope_parameters and hasattr(config, "rope_theta"):
rope_parameters["rope_theta"] = config.rope_theta
if "partial_rotary_factor" not in rope_parameters and hasattr(
config, "partial_rotary_factor"
):
rope_parameters["partial_rotary_factor"] = config.partial_rotary_factor
rope_scaling = getattr(config, "rope_scaling", None)
if isinstance(rope_scaling, dict):
rope_scaling = copy.deepcopy(rope_scaling)
if "type" in rope_scaling and "rope_type" not in rope_scaling:
rope_scaling["rope_type"] = rope_scaling.pop("type")
rope_parameters.update(rope_scaling)
return rope_parameters or None
def clear_linear_attention_cache_for_new_sequences(
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
) -> None:
num_prefills = getattr(attn_metadata, "num_prefills", 0)
if num_prefills <= 0:
return
num_decodes = getattr(attn_metadata, "num_decodes", 0)
prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None)
for prefill_idx in range(num_prefills):
if num_decodes + prefill_idx + 1 >= len(attn_metadata.query_start_loc):
break
q_start = attn_metadata.query_start_loc[num_decodes + prefill_idx]
q_end = attn_metadata.query_start_loc[num_decodes + prefill_idx + 1]
query_len = q_end - q_start
context_len = attn_metadata.seq_lens[num_decodes + prefill_idx] - query_len
if context_len == 0:
if prefill_state_indices is not None:
block_to_clear = prefill_state_indices[prefill_idx]
else:
block_to_clear = state_indices_tensor[num_decodes + prefill_idx]
kv_cache[block_to_clear, ...] = 0
def linear_attention_prefill_and_mix(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
slope_rate: torch.Tensor,
block_size: int,
decode_fn,
prefix_fn,
layer_idx: int | None = None,
) -> torch.Tensor:
hidden = []
req_offset = getattr(attn_metadata, "num_decodes", 0)
prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None)
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
if req_offset + _prefill_idx + 1 >= len(attn_metadata.query_start_loc):
break
if prefill_state_indices is not None and _prefill_idx >= len(
prefill_state_indices
):
break
if prefill_state_indices is None and _prefill_idx >= len(state_indices_tensor):
break
_start = attn_metadata.query_start_loc[req_offset + _prefill_idx]
_end = attn_metadata.query_start_loc[req_offset + _prefill_idx + 1]
if prefill_state_indices is not None:
slot_id = prefill_state_indices[_prefill_idx]
else:
slot_id = state_indices_tensor[req_offset + _prefill_idx]
qs = q[_start:_end].transpose(0, 1).contiguous()
ks = k[_start:_end].transpose(0, 1).contiguous()
vs = v[_start:_end].transpose(0, 1).contiguous()
slice_layer_cache = kv_cache[slot_id, ...]
out_slice = prefix_fn(
qs,
ks,
vs,
slice_layer_cache,
slope_rate,
block_size,
layer_idx=layer_idx,
)
hidden.append(out_slice.contiguous())
if attn_metadata.num_decode_tokens > 0:
hidden_decode = decode_fn(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
hidden.insert(0, hidden_decode)
if not hidden:
return torch.empty((0, q.size(1) * q.size(2)), device=q.device, dtype=q.dtype)
hidden = torch.concat(hidden, dim=0).contiguous()
return hidden
@triton.jit
def _bailing_linear_attn_decode_spec_step_kernel(
q_ptr,
k_ptr,
v_ptr,
kv_cache_ptr,
slope_rate_ptr,
state_indices_ptr,
query_start_loc_ptr,
num_accepted_tokens_ptr,
output_ptr,
q_start: tl.constexpr,
D: tl.constexpr,
q_b_stride,
q_h_stride,
q_d_stride,
k_b_stride,
k_h_stride,
k_d_stride,
v_b_stride,
v_h_stride,
v_d_stride,
cache_b_stride,
cache_h_stride,
cache_d0_stride,
cache_d1_stride,
state_indices_b_stride,
state_indices_t_stride,
output_b_stride,
output_d_stride,
DRAFT_IDX: tl.constexpr,
STATE_WIDTH: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
req_id = tl.program_id(0)
head_id = tl.program_id(1)
block_id = tl.program_id(2)
req_start = tl.load(query_start_loc_ptr + req_id).to(tl.int64)
req_end = tl.load(query_start_loc_ptr + req_id + 1).to(tl.int64)
query_len = req_end - req_start
if query_len <= DRAFT_IDX:
return
dst_slot = tl.load(
state_indices_ptr
+ req_id * state_indices_b_stride
+ DRAFT_IDX * state_indices_t_stride
).to(tl.int64)
if dst_slot == -1:
return
if DRAFT_IDX == 0:
accepted_offset = tl.load(num_accepted_tokens_ptr + req_id).to(tl.int64) - 1
accepted_offset = tl.maximum(accepted_offset, 0)
accepted_offset = tl.minimum(accepted_offset, STATE_WIDTH - 1)
src_slot = tl.load(
state_indices_ptr
+ req_id * state_indices_b_stride
+ accepted_offset * state_indices_t_stride
).to(tl.int64)
else:
src_slot = tl.load(
state_indices_ptr
+ req_id * state_indices_b_stride
+ (DRAFT_IDX - 1) * state_indices_t_stride
).to(tl.int64)
if src_slot == -1:
return
token_idx = req_start - q_start + DRAFT_IDX
qk_offsets = tl.arange(0, D)
v_offsets = tl.arange(0, BLOCK_SIZE) + block_id * BLOCK_SIZE
qk_mask = qk_offsets < D
v_mask = v_offsets < D
kv_mask = qk_mask[:, None] & v_mask[None, :]
q = tl.load(
q_ptr + token_idx * q_b_stride + head_id * q_h_stride + qk_offsets * q_d_stride,
mask=qk_mask,
other=0.0,
)
k = tl.load(
k_ptr + token_idx * k_b_stride + head_id * k_h_stride + qk_offsets * k_d_stride,
mask=qk_mask,
other=0.0,
)
v = tl.load(
v_ptr + token_idx * v_b_stride + head_id * v_h_stride + v_offsets * v_d_stride,
mask=v_mask,
other=0.0,
)
cache_offsets = (
qk_offsets[:, None] * cache_d0_stride + v_offsets[None, :] * cache_d1_stride
)
src_cache_ptr = (
kv_cache_ptr
+ src_slot * cache_b_stride
+ head_id * cache_h_stride
+ cache_offsets
)
dst_cache_ptr = (
kv_cache_ptr
+ dst_slot * cache_b_stride
+ head_id * cache_h_stride
+ cache_offsets
)
slope = tl.load(slope_rate_ptr + head_id)
decay = tl.exp(-slope)
kv_old = tl.load(src_cache_ptr, mask=kv_mask, other=0.0)
kv_new = k[:, None] * v[None, :] + decay * kv_old
output = tl.sum(q[:, None].to(tl.float32) * kv_new, axis=0)
tl.store(dst_cache_ptr, kv_new, mask=kv_mask)
tl.store(
output_ptr
+ token_idx * output_b_stride
+ (head_id * D + v_offsets) * output_d_stride,
output,
mask=v_mask,
)
def bailing_linear_attention_decode_spec(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
slope_rate: torch.Tensor,
state_indices_tensor: torch.Tensor,
query_start_loc: torch.Tensor,
num_accepted_tokens: torch.Tensor,
q_start: int,
q_end: int | None,
slot_start: int,
slot_end: int | None,
block_size: int,
) -> torch.Tensor:
q_decode = q[q_start:q_end]
k_decode = k[q_start:q_end]
v_decode = v[q_start:q_end]
hidden = torch.empty(
(q_decode.shape[0], q.shape[1] * q.shape[2]),
device=q.device,
dtype=q.dtype,
)
hidden.zero_()
state_indices_tensor = state_indices_tensor[slot_start:slot_end]
query_start_loc = query_start_loc.to(device=q.device)
batch_size = state_indices_tensor.shape[0]
num_heads = q_decode.shape[1]
head_dim = q_decode.shape[2]
assert k_decode.shape == (q_decode.shape[0], num_heads, head_dim)
assert v_decode.shape == (q_decode.shape[0], num_heads, head_dim)
state_width = state_indices_tensor.shape[1]
grid = (batch_size, num_heads, triton.cdiv(head_dim, block_size))
for draft_idx in range(state_width):
_bailing_linear_attn_decode_spec_step_kernel[grid](
q_decode,
k_decode,
v_decode,
kv_cache,
slope_rate,
state_indices_tensor,
query_start_loc,
num_accepted_tokens[:batch_size],
hidden,
q_start,
head_dim,
q_decode.stride(0),
q_decode.stride(1),
q_decode.stride(2),
k_decode.stride(0),
k_decode.stride(1),
k_decode.stride(2),
v_decode.stride(0),
v_decode.stride(1),
v_decode.stride(2),
kv_cache.stride(0),
kv_cache.stride(1),
kv_cache.stride(2),
kv_cache.stride(3),
state_indices_tensor.stride(0),
state_indices_tensor.stride(1),
hidden.stride(0),
hidden.stride(1),
DRAFT_IDX=draft_idx,
STATE_WIDTH=state_width,
BLOCK_SIZE=block_size,
)
return hidden
class BailingGroupRMSNormGate(RMSNormGated):
def __init__(
self,
hidden_size,
eps=1e-5,
group_size=None,
norm_before_gate=True,
device=None,
dtype=None,
):
super().__init__(
hidden_size,
eps=eps,
group_size=group_size,
norm_before_gate=norm_before_gate,
device=device,
dtype=dtype,
activation="sigmoid",
)
# Add custom weight loader for TP sharding
self.weight.weight_loader = self._weight_loader
@staticmethod
def _weight_loader(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
"""Load weight with TP sharding."""
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
shard_size = loaded_weight.shape[0] // tp_size
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
param.data.copy_(loaded_weight[shard].contiguous())
# --8<-- [start:bailing_moe_linear_attention]
@PluggableLayer.register("bailing_moe_linear_attention")
class BailingMoELinearAttention(LinearAttention):
"""Pluggable Bailing MoE Linear Attention layer which allows OOT backends
to add custom implementations.
This implements the linear attention mechanism from sglang, adapted for
vLLM's v1 engine with MambaBase interface support.
"""
# --8<-- [end:bailing_moe_linear_attention]
def __init__(
self,
config: PreTrainedConfig,
vllm_config: VllmConfig,
prefix: str = "linear_attn",
):
super().__init__(config, vllm_config, prefix)
self.scaling = self.head_dim**-0.5
self.tp_heads = self.num_heads // self.tp_size
self.max_position_embeddings = config.max_position_embeddings
self.rope_theta = getattr(config, "rope_theta", 600000)
self.tp_kv_heads = self.num_heads // self.tp_size
self.q_size_per_rank = self.head_dim * self.tp_heads
self.kv_size_per_rank = self.head_dim * self.tp_kv_heads
self.use_qk_norm = getattr(config, "use_qk_norm", False)
self.linear_backend = "minimax"
self.linear_scale = self.linear_backend == "minimax"
self.linear_rope = getattr(config, "linear_rope", True)
if hasattr(config, "use_linear_silu"):
self.linear_silu = config.use_linear_silu
elif hasattr(config, "linear_silu"):
self.linear_silu = config.linear_silu
else:
self.linear_silu = False
self.query_key_value = QKVParallelLinear(
self.hidden_size,
self.head_dim,
self.num_heads,
self.num_heads, # MHA: kv_heads = num_heads
bias=(config.use_bias or config.use_qkv_bias),
quant_config=self.quant_config,
prefix=f"{prefix}.query_key_value",
)
if self.use_qk_norm:
self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.g_proj = ColumnParallelLinear(
self.hidden_size,
self.hidden_inner_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_proj",
)
self.dense = RowParallelLinear(
self.hidden_inner_size,
self.hidden_size,
bias=config.use_bias,
quant_config=self.quant_config,
prefix=f"{prefix}.dense",
reduce_results=True,
)
self.group_norm_size = getattr(config, "group_norm_size", 1)
self.rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
assert self.tp_size <= self.group_norm_size, (
"tp_size must be <= group_norm_size for local rms norm"
)
assert self.group_norm_size % self.tp_size == 0, (
"group_norm_size must be divisible by tp_size"
)
# When group_norm_size == 1, group_size equals hidden_size // tp_size
self.g_norm = BailingGroupRMSNormGate(
hidden_size=self.hidden_inner_size // self.tp_size,
eps=self.rms_norm_eps,
group_size=(
self.hidden_inner_size // self.group_norm_size
if self.group_norm_size > 1
else self.hidden_inner_size // self.tp_size
),
)
# use fp32 rotary embedding
rope_parameters = _build_rope_parameters(config)
self.rotary_emb = get_rope(
self.head_dim,
max_position=self.max_position_embeddings,
is_neox_style=True,
rope_parameters=rope_parameters or None,
)
# Build slope tensor for linear attention decay
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
if self.num_hidden_layers <= 1:
self.slope_rate = slope_rate * (1 + 1e-5)
else:
self.slope_rate = slope_rate * (
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
)
self.tp_slope = self.slope_rate[
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
].contiguous()
# Register for compilation
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def get_attn_backend(self):
from vllm.v1.attention.backends.linear_attn import (
BailingLinearAttentionBackend,
)
return BailingLinearAttentionBackend
@staticmethod
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
"""Load weight for linear attention layers.
For FP8 quantized parameters, we need to use the weight_loader if available,
as it handles special cases like tensor parallelism sharding.
"""
# Check if param has a weight_loader (for vLLM ModelWeightParameter)
weight_loader = getattr(param, "weight_loader", None)
if weight_loader is not None:
# Use the weight_loader which handles TP sharding and quantization
weight_loader(param, loaded_weight)
else:
# Fall back to direct copy for standard tensors
assert param.size() == loaded_weight.size(), (
f"Shape mismatch: {param.shape} vs {loaded_weight.shape}"
)
param.data.copy_(loaded_weight)
def forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
positions: torch.Tensor,
) -> None:
"""Forward method called by torch.ops.vllm.linear_attention."""
torch.ops.vllm.linear_attention(
hidden_states,
output,
positions,
self.prefix,
)
def _forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
positions: torch.Tensor,
) -> None:
"""Actual forward implementation."""
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is not None:
assert isinstance(attn_metadata, dict)
attn_metadata = attn_metadata.get(self.prefix) # type: ignore
if attn_metadata is not None:
assert isinstance(attn_metadata, LinearAttentionMetadata)
num_actual_tokens = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
)
else:
num_actual_tokens = hidden_states.shape[0]
# QKV projection
qkv, _ = self.query_key_value(hidden_states[:num_actual_tokens])
# use rotary_emb support fp32
qkv = qkv.to(torch.float32)
if self.linear_silu:
qkv = F.silu(qkv)
# Split q, k, v
q, k, v = torch.split(
qkv,
[self.q_size_per_rank, self.kv_size_per_rank, self.kv_size_per_rank],
dim=-1,
)
# Apply QK norm if needed
if self.use_qk_norm:
q = q.reshape(-1, self.tp_heads, self.head_dim)
k = k.reshape(-1, self.tp_kv_heads, self.head_dim)
q = layernorm_fn(
q,
self.query_layernorm.weight.data,
bias=None,
eps=self.rms_norm_eps,
is_rms_norm=True,
)
k = layernorm_fn(
k,
self.key_layernorm.weight.data,
bias=None,
eps=self.rms_norm_eps,
is_rms_norm=True,
)
q = q.reshape(-1, self.q_size_per_rank)
k = k.reshape(-1, self.kv_size_per_rank)
# Apply rotary embeddings
if self.linear_rope:
q, k = self.rotary_emb(positions[:num_actual_tokens], q, k)
# Reshape to [batch, heads, seq_len, head_dim]
q = q.view((qkv.shape[0], self.tp_heads, self.head_dim))
k = k.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
v = v.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
# Apply scaling if using minimax backend
if self.linear_scale:
q = q * self.scaling
# Get KV cache and state indices
if attn_metadata is not None:
kv_cache = self.kv_cache[0]
state_indices_tensor = attn_metadata.state_indices_tensor
clear_linear_attention_cache_for_new_sequences(
kv_cache, state_indices_tensor, attn_metadata
)
# Compute attention
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
if attn_metadata is None:
hidden = torch.empty(
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
)
else:
if not decode_only:
hidden = self._prefill_and_mix_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
else:
hidden = self._decode_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
# Apply group norm and gate (matching SGLang behavior)
gate, _ = self.g_proj(hidden_states[:num_actual_tokens])
if self.group_norm_size > 1:
hidden = self.g_norm(hidden, gate)
else:
hidden = self.g_norm(hidden)
hidden = F.sigmoid(gate) * hidden
hidden = hidden.to(hidden_states.dtype)
# Output projection
dense_out, _ = self.dense(hidden)
output[:num_actual_tokens] = dense_out
def _prefill_and_mix_infer(
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
):
"""Handle prefill (mixed with decode if any)."""
return linear_attention_prefill_and_mix(
q=q,
k=k,
v=v,
kv_cache=kv_cache,
state_indices_tensor=state_indices_tensor,
attn_metadata=attn_metadata,
slope_rate=self.tp_slope,
block_size=self.BLOCK,
decode_fn=self._decode_infer,
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
layer_idx=self.layer_idx,
)
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
"""Handle decode (single token per sequence)."""
decode_state_indices = getattr(attn_metadata, "state_indices_tensor_d", None)
num_accepted_tokens = getattr(attn_metadata, "num_accepted_tokens", None)
query_start_loc = getattr(attn_metadata, "query_start_loc_d", None)
if (
decode_state_indices is not None
and decode_state_indices.dim() > 1
and num_accepted_tokens is not None
and query_start_loc is not None
):
return bailing_linear_attention_decode_spec(
q,
k,
v,
kv_cache,
self.tp_slope,
decode_state_indices,
query_start_loc,
num_accepted_tokens,
q_start=0,
q_end=attn_metadata.num_decode_tokens,
slot_start=0,
slot_end=attn_metadata.num_decodes,
block_size=32,
)
decode_state_indices = (
state_indices_tensor
if decode_state_indices is None
else decode_state_indices
)
hidden = linear_attention_decode(
q,
k,
v,
kv_cache,
self.tp_slope,
decode_state_indices,
q_start=0,
q_end=attn_metadata.num_decode_tokens,
slot_start=0,
slot_end=attn_metadata.num_decodes,
block_size=32,
)
return hidden
@@ -1,66 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from transformers import PreTrainedConfig
from vllm.config import (
VllmConfig,
)
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
)
from vllm.model_executor.models.utils import extract_layer_index
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
class LinearAttention(PluggableLayer, MambaBase):
"""Base class for Linear attention layer."""
def __init__(
self, config: PreTrainedConfig, vllm_config: VllmConfig, prefix: str = ""
):
super().__init__()
self.layer_idx = extract_layer_index(prefix)
self.prefix = prefix
self.model_config = vllm_config.model_config
self.cache_config = vllm_config.cache_config
self.quant_config = vllm_config.quant_config
self.BLOCK = getattr(config, "block", 256)
self.hidden_size = config.hidden_size
self.num_heads = config.num_attention_heads
self.num_hidden_layers = config.num_hidden_layers
self.head_dim = (
config.head_dim
if hasattr(config, "head_dim")
else config.hidden_size // self.num_heads
)
self.hidden_inner_size = self.head_dim * self.num_heads
self.tp_size = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()
assert self.num_heads % self.tp_size == 0
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.LINEAR
def get_state_dtype(self) -> tuple[torch.dtype]:
assert self.model_config is not None
assert self.cache_config is not None
return MambaStateDtypeCalculator.linear_attention_state_dtype(
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
)
def get_state_shape(self) -> tuple[tuple[int, int, int], ...]:
return MambaStateShapeCalculator.linear_attention_state_shape(
num_heads=self.num_heads, tp_size=self.tp_size, head_dim=self.head_dim
)
@@ -1,345 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Callable
import torch
import torch.nn.functional as F
from einops import rearrange
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.lightning_attn import (
lightning_attention,
linear_decode_forward_triton,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
from vllm.model_executor.layers.minimax_rms_norm import MiniMaxText01RMSNormTP
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
def clear_linear_attention_cache_for_new_sequences(
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
) -> None:
num_prefills = getattr(attn_metadata, "num_prefills", 0)
if num_prefills <= 0:
return
num_decode_tokens = getattr(attn_metadata, "num_decode_tokens", 0)
for prefill_idx in range(num_prefills):
q_start = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx]
q_end = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx + 1]
query_len = q_end - q_start
context_len = (
attn_metadata.seq_lens[num_decode_tokens + prefill_idx] - query_len
)
if context_len == 0:
block_to_clear = state_indices_tensor[num_decode_tokens + prefill_idx]
kv_cache[block_to_clear, ...] = 0
def linear_attention_decode(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
slope_rate: torch.Tensor,
state_indices_tensor: torch.Tensor,
q_start: int = 0,
q_end: int | None = None,
slot_start: int = 0,
slot_end: int | None = None,
block_size: int = 32,
) -> torch.Tensor:
q = q[q_start:q_end].unsqueeze(2).contiguous()
k = k[q_start:q_end].unsqueeze(2).contiguous()
v = v[q_start:q_end].unsqueeze(2).contiguous()
slot_id = state_indices_tensor[slot_start:slot_end]
return linear_decode_forward_triton(
q, k, v, kv_cache, slope_rate, slot_id, block_size
)
def linear_attention_prefill_and_mix(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
slope_rate: torch.Tensor,
block_size: int,
decode_fn: Callable[..., torch.Tensor],
prefix_fn: Callable[..., torch.Tensor],
layer_idx: int | None = None,
) -> torch.Tensor:
hidden = []
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
if _prefill_idx >= len(attn_metadata.query_start_loc):
break
if _prefill_idx >= len(state_indices_tensor):
break
offset = attn_metadata.num_decode_tokens
_start = attn_metadata.query_start_loc[offset + _prefill_idx]
_end = attn_metadata.query_start_loc[offset + _prefill_idx + 1]
slot_id = state_indices_tensor[offset + _prefill_idx]
qs = q[_start:_end].transpose(0, 1).contiguous()
ks = k[_start:_end].transpose(0, 1).contiguous()
vs = v[_start:_end].transpose(0, 1).contiguous()
slice_layer_cache = kv_cache[slot_id, ...]
out_slice = prefix_fn(
qs,
ks,
vs,
slice_layer_cache,
slope_rate,
block_size,
layer_idx=layer_idx,
)
hidden.append(out_slice.contiguous())
if attn_metadata.num_decode_tokens > 0:
hidden_decode = decode_fn(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
hidden.insert(0, hidden_decode)
if not hidden:
return torch.empty((0, q.size(-1)), device=q.device, dtype=q.dtype)
hidden = torch.concat(hidden, dim=0).contiguous()
return hidden
class MiniMaxText01LinearKernel:
@staticmethod
def jit_linear_forward_prefix(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_caches: torch.Tensor,
slope_rate: torch.Tensor,
block_size: int,
layer_idx: int | None = None,
**kwargs,
) -> torch.Tensor:
slope_rate = slope_rate.to(torch.float32)
should_pad_dim = q.dim() == 3
if should_pad_dim:
q = q.unsqueeze(0)
k = k.unsqueeze(0)
v = v.unsqueeze(0)
b, h, n, d = q.shape
e = d
kv_history = kv_caches.reshape(1, h, d, e).contiguous()
output, kv_history = lightning_attention(
q, k, v, slope_rate, block_size=block_size, kv_history=kv_history
)
kv_caches.copy_(kv_history[:, :, -1, :, :].reshape(h, d, e))
assert output.shape[0] == 1, "batch size must be 1"
return rearrange(output.squeeze(0), "h n d -> n (h d)")
@PluggableLayer.register("minimax_text_01_attention")
class MiniMaxText01LinearAttention(LinearAttention):
def __init__(
self,
config,
vllm_config,
prefix: str = "linear_attn",
) -> None:
super().__init__(config, vllm_config, prefix)
self.tp_heads = self.num_heads // self.tp_size
self.qkv_size = self.num_heads * self.head_dim
self.tp_hidden = self.head_dim * self.tp_heads
self.qkv_proj = ColumnParallelLinear(
self.hidden_size,
self.hidden_inner_size * 3,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.output_gate = ColumnParallelLinear(
self.hidden_size,
self.hidden_inner_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.output_gate",
)
self.out_proj = RowParallelLinear(
self.hidden_inner_size,
self.hidden_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.out_proj",
)
self.norm = MiniMaxText01RMSNormTP(
self.hidden_inner_size,
eps=1e-5,
)
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
if self.num_hidden_layers <= 1:
self.slope_rate = slope_rate * (1 + 1e-5)
else:
self.slope_rate = slope_rate * (
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
)
self.tp_slope = self.slope_rate[
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
].contiguous()
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
@staticmethod
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
assert param.size() == loaded_weight.size()
param.data.copy_(loaded_weight)
return
@staticmethod
def _build_slope_tensor(n_attention_heads: int):
def get_slopes(n):
def get_slopes_power_of_2(n):
start = 2 ** (-(2 ** -(math.log2(n) - 3)))
ratio = start
return [start * ratio**i for i in range(n)]
if math.log2(n).is_integer():
return get_slopes_power_of_2(n)
else:
closest_power_of_2 = 2 ** math.floor(math.log2(n))
return (
get_slopes_power_of_2(closest_power_of_2)
+ get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2]
)
slopes = torch.tensor(
get_slopes(n_attention_heads), dtype=torch.float32
).reshape(n_attention_heads, 1, 1)
return slopes
def _prefill_and_mix_infer(
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
):
return linear_attention_prefill_and_mix(
q=q,
k=k,
v=v,
kv_cache=kv_cache,
state_indices_tensor=state_indices_tensor,
attn_metadata=attn_metadata,
slope_rate=self.tp_slope,
block_size=self.BLOCK,
decode_fn=self._decode_infer,
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
layer_idx=self.layer_idx,
)
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
hidden = linear_attention_decode(
q,
k,
v,
kv_cache,
self.tp_slope,
state_indices_tensor,
q_start=0,
q_end=attn_metadata.num_decode_tokens,
slot_start=0,
slot_end=attn_metadata.num_decodes,
block_size=32,
)
return hidden
def forward(
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
) -> None:
torch.ops.vllm.linear_attention(
hidden_states,
output,
positions,
self.prefix,
)
def _forward(
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, LinearAttentionMetadata)
num_actual_tokens = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
)
else:
num_actual_tokens = hidden_states.shape[0]
qkv, _ = self.qkv_proj(hidden_states[:num_actual_tokens])
qkv32 = qkv.to(torch.float32)
qkvact = torch.nn.functional.silu(qkv32)
qkvact = qkvact.view((qkv.shape[0], self.tp_heads, -1))
q, k, v = torch.split(qkvact, [self.head_dim] * 3, dim=-1)
if attn_metadata is not None:
kv_cache = self.kv_cache[0]
state_indices_tensor = attn_metadata.state_indices_tensor
clear_linear_attention_cache_for_new_sequences(
kv_cache, state_indices_tensor, attn_metadata
)
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
if attn_metadata is None:
hidden = torch.empty(
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
)
else:
if not decode_only:
hidden = self._prefill_and_mix_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
else:
hidden = self._decode_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
hidden = self.norm._forward(hidden)
gate, _ = self.output_gate(hidden_states[:num_actual_tokens])
hidden = F.sigmoid(gate) * hidden
hidden = hidden.to(hidden_states.dtype)
output[:num_actual_tokens], _ = self.out_proj(hidden)
@eager_break_during_capture
def linear_attention(
hidden_states: torch.Tensor,
output: torch.Tensor,
positions: torch.Tensor,
layer_name: str,
) -> None:
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._forward(hidden_states=hidden_states, output=output, positions=positions)
direct_register_custom_op(
op_name="linear_attention",
op_func=linear_attention,
mutates_args=["output"],
)
@@ -1,485 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import NamedTuple
import torch
from torch import nn
from torch.nn.parameter import Parameter
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_scan_fn
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
# --8<-- [start:mamba_mixer]
@PluggableLayer.register("mamba_mixer")
class MambaMixer(MambaBase, PluggableLayer):
"""Compute ∆, A, B, C, and D the state space parameters and compute
the `contextualized_states`. A, D are input independent
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
for why A isn't selective) ∆, B, C are input-dependent
(this is a key difference between Mamba and the linear time
invariant S4, and is why Mamba is called
**selective** state spaces)
"""
# --8<-- [end:mamba_mixer]
def __init__(
self,
hidden_size: int,
ssm_state_size: int,
conv_kernel_size: int,
intermediate_size: int,
time_step_rank: int,
use_conv_bias: bool,
use_bias: bool,
use_rms_norm: bool,
rms_norm_has_weight: bool = True,
rms_norm_eps: float = 1e-5,
activation="silu",
is_lora_enabled: bool = False,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
):
super().__init__()
self.time_step_rank = time_step_rank
self.ssm_state_size = ssm_state_size
self.use_rms_norm = use_rms_norm
self.activation = activation
self.is_lora_enabled = is_lora_enabled
self.conv_kernel_size = conv_kernel_size
self.intermediate_size = intermediate_size
self.conv1d = ColumnParallelLinear(
input_size=conv_kernel_size,
output_size=intermediate_size,
bias=use_conv_bias,
prefix=f"{prefix}.conv1d",
)
# unsqueeze to fit conv1d weights shape into the linear weights shape.
# Can't do this in `weight_loader` since it already exists in
# `ColumnParallelLinear` and `set_weight_attrs`
# doesn't allow to override it
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
self.in_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=use_bias,
prefix=f"{prefix}.in_proj",
)
# selective projection used to make dt, B and C input dependent
self.x_proj = RowParallelLinear(
intermediate_size,
time_step_rank + ssm_state_size * 2,
bias=False,
prefix=f"{prefix}.x_proj",
)
# time step projection (discretization) -
# In the forward we need to apply dt_proj without the bias,
# as the bias is added in the selective scan kernel.
self.dt_proj = ColumnParallelLinear(
time_step_rank,
intermediate_size,
bias=True,
skip_bias_add=True,
prefix=f"{prefix}.dt_proj",
)
def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
tp_rank = get_tensor_model_parallel_rank()
tp_size = get_tensor_model_parallel_world_size()
param.data.copy_(
loaded_weight.data.split(loaded_weight.shape[0] // tp_size, dim=0)[
tp_rank
]
)
def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
weight_loader(param, -torch.exp(loaded_weight.float()))
tp_size = get_tensor_model_parallel_world_size()
self.A = nn.Parameter(
torch.empty(
intermediate_size // tp_size,
ssm_state_size,
dtype=torch.float32,
)
)
self.D = nn.Parameter(torch.ones(intermediate_size // tp_size))
set_weight_attrs(self.D, {"weight_loader": weight_loader})
set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
self.out_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=use_bias,
input_is_parallel=True,
prefix=f"{prefix}.out_proj",
)
self.dt_layernorm = (
RMSNorm(
time_step_rank,
eps=rms_norm_eps,
has_weight=rms_norm_has_weight,
)
if use_rms_norm
else None
)
self.b_layernorm = (
RMSNorm(
ssm_state_size,
eps=rms_norm_eps,
has_weight=rms_norm_has_weight,
)
if use_rms_norm
else None
)
self.c_layernorm = (
RMSNorm(
ssm_state_size,
eps=rms_norm_eps,
has_weight=rms_norm_has_weight,
)
if use_rms_norm
else None
)
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
# The inner tuple is (conv_state, ssm_state)
self.kv_cache = (torch.tensor([]), torch.tensor([]))
self.model_config = model_config
self.cache_config = cache_config
self.prefix = prefix
def _ssm_transform(
self, x: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# LoRA kernel requires contiguous tensor.
# ROCm: Non-contiguous tensors cause incorrect GEMM
# results when batch > 1.
if self.is_lora_enabled or current_platform.is_rocm():
x = x.contiguous()
ssm_params = self.x_proj(x)[0]
time_step, B, C = torch.split(
ssm_params,
[self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
dim=-1,
)
if self.use_rms_norm:
assert self.dt_layernorm is not None
assert self.b_layernorm is not None
assert self.c_layernorm is not None
time_step = self.dt_layernorm(time_step.contiguous())
B = self.b_layernorm(B.contiguous())
C = self.c_layernorm(C.contiguous())
# ROCm: tensor from split is non-contiguous, causing incorrect
# GEMM results in dt_proj.
if current_platform.is_rocm():
time_step = time_step.contiguous()
discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
return discrete_time_step, B, C
def forward(self, hidden_states: torch.Tensor, output: torch.Tensor):
torch.ops.vllm.mamba_mixer(
hidden_states,
output,
_encode_layer_name(self.prefix),
)
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
"""Run the Mamba-1 SSM pipeline.
Steps
-----
1. Apply the gated-MLP linear projection to the raw input.
2. Pass the projected sequence through the convolutional mixing layer.
3. Feed the result into the State-Space Model (SSM) blocks.
4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
to produce contextual representations.
5. Project the contextualised sequence back
to the output embedding dimension.
Batch handling
--------------
Prefill and decode tokens are processed by dedicated CUDA
kernels for both the convolutional (conv1d) and SSM stages.
In the case of a mixed batch (containing both prefill and
decode tokens), both sets of kernels are executed independently
and their outputs are concatenated before the final output projection.
"""
forward_context: ForwardContext = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, Mamba1AttentionMetadata)
query_start_loc_p = attn_metadata.query_start_loc_p
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
conv_state = (
self.kv_cache[0]
if is_conv_state_dim_first()
else self.kv_cache[0].transpose(-1, -2)
)
ssm_state = self.kv_cache[1]
has_initial_states_p = attn_metadata.has_initial_states_p
cu_chunk_seqlen_p = attn_metadata.cu_chunk_seqlen_p
last_chunk_indices_p = attn_metadata.last_chunk_indices_p
# 1. Gated MLP's linear projection
projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
hidden_states_BC, gate = projected_states.chunk(2, dim=-2)
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
)
if attn_metadata is None:
# V1 profile run
hidden_states_BC = hidden_states_BC.contiguous()
return self.out_proj(hidden_states_BC.transpose(-2, -1))[0]
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
num_decode_tokens = attn_metadata.num_decode_tokens
has_prefill = num_prefill_tokens > 0
has_decode = num_decode_tokens > 0
num_actual_tokens = num_prefill_tokens + num_decode_tokens
prefill_decode_split = split_batch_to_prefill_and_decode(
hidden_states_BC,
gate,
num_prefill_tokens,
num_decode_tokens,
)
hidden_states_BC_p = prefill_decode_split.hidden_states_BC_p
hidden_states_BC_d = prefill_decode_split.hidden_states_BC_d
gate_p = prefill_decode_split.gate_p
gate_d = prefill_decode_split.gate_d
ssm_outputs = []
if has_prefill:
# 2. Convolution sequence transformation
conv_out_p = causal_conv1d_fn(
hidden_states_BC_p,
conv_weights,
self.conv1d.bias,
activation=self.activation,
conv_states=conv_state,
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
query_start_loc=query_start_loc_p,
metadata=attn_metadata,
)
# 3. State Space Model sequence transformations.
discrete_time_step_p, B_p, C_p = self._ssm_transform(
conv_out_p.transpose(-2, -1)
)
time_proj_bias = self._time_proj_bias()
# 4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
scan_out_p = selective_scan_fn(
conv_out_p,
ssm_state,
discrete_time_step_p,
self.A,
B_p.transpose(-2, -1),
C_p.transpose(-2, -1),
self.D.float(),
gate_p,
time_proj_bias,
delta_softplus=True,
cache_indices=state_indices_tensor_p,
has_initial_state=has_initial_states_p,
query_start_loc=query_start_loc_p,
cu_chunk_seqlen=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
)
ssm_outputs.append(scan_out_p)
if has_decode:
# state_indices_tensor_d is assigned when attn_metadata is not None,
# and has_decode is only True when attn_metadata is not None
assert state_indices_tensor_d is not None
# 2. Convolution sequence transformation
conv_out_d = causal_conv1d_update(
hidden_states_BC_d.transpose(0, 1),
conv_state,
conv_weights,
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
).transpose(0, 1)
# 3. State Space Model sequence transformation.
discrete_time_step_d, B_d, C_d = self._ssm_transform(
conv_out_d.transpose(-2, -1)
)
time_proj_bias = self._time_proj_bias()
# 4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
scan_outputs_d = torch.empty_like(hidden_states_BC_d.transpose(0, 1))
selective_state_update(
ssm_state,
conv_out_d.transpose(0, 1),
discrete_time_step_d.transpose(0, 1),
self.A,
B_d,
C_d,
self.D,
time_proj_bias,
z=gate_d.transpose(0, 1),
dt_softplus=True,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=scan_outputs_d,
)
scan_outputs_d = scan_outputs_d.transpose(0, 1)
ssm_outputs.insert(0, scan_outputs_d)
scan_outputs_combined = (
ssm_outputs[0] if len(ssm_outputs) == 1 else torch.cat(ssm_outputs, dim=-1)
)
# 5. Final output projection
if self.is_lora_enabled: # Lora kernel requires contiguous tensor.
scan_outputs_combined = scan_outputs_combined.transpose(-2, -1).contiguous()
out = self.out_proj(scan_outputs_combined)[0]
else:
out = self.out_proj(scan_outputs_combined.transpose(-2, -1))[0]
output[:num_actual_tokens] = out
def get_state_dtype(self) -> tuple[torch.dtype]:
assert self.model_config is not None
assert self.cache_config is not None
return MambaStateDtypeCalculator.mamba1_state_dtype(
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
self.cache_config.mamba_ssm_cache_dtype,
)
def get_state_shape(self) -> tuple[tuple[int, ...], tuple[int, ...]]:
return MambaStateShapeCalculator.mamba1_state_shape(
tp_world_size=get_tensor_model_parallel_world_size(),
intermediate_size=self.intermediate_size,
state_size=self.ssm_state_size,
conv_kernel=self.conv_kernel_size,
)
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.MAMBA1
def _time_proj_bias(self) -> torch.Tensor | None:
if hasattr(self.dt_proj, "bias") and self.dt_proj.bias is not None:
return self.dt_proj.bias.float()
return None
class PrefillDecodeSplit(NamedTuple):
hidden_states_BC_p: torch.Tensor
hidden_states_BC_d: torch.Tensor
gate_p: torch.Tensor
gate_d: torch.Tensor
def split_batch_to_prefill_and_decode(
hidden_states_BC: torch.Tensor,
gate: torch.Tensor,
num_prefill_tokens: int,
num_decode_tokens: int,
) -> PrefillDecodeSplit:
num_actual_tokens = num_prefill_tokens + num_decode_tokens
# In v1, decode tokens come first, then prefill tokens.
hidden_states_BC_d, hidden_states_BC_p = torch.split(
hidden_states_BC[..., :num_actual_tokens],
[num_decode_tokens, num_prefill_tokens],
dim=-1,
)
gate_d, gate_p = torch.split(
gate[..., :num_actual_tokens], [num_decode_tokens, num_prefill_tokens], dim=-1
)
return PrefillDecodeSplit(
hidden_states_BC_p=hidden_states_BC_p,
hidden_states_BC_d=hidden_states_BC_d,
gate_p=gate_p,
gate_d=gate_d,
)
@eager_break_during_capture
def mamba_mixer(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_name: LayerNameType,
) -> None:
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self.forward_impl(hidden_states=hidden_states, output=output)
direct_register_custom_op(
op_name="mamba_mixer",
op_func=mamba_mixer,
mutates_args=["output"],
)
@@ -1,83 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@triton.jit
def _gather_initial_states_kernel(
state_ptr,
indices_ptr,
has_initial_state_ptr,
output_ptr,
stride_state_batch,
stride_indices,
stride_has_initial_state,
row_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
):
block_idx = tl.program_id(0)
batch_idx = tl.program_id(1)
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < row_size
if launch_pdl:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
has_initial_state = tl.load(
has_initial_state_ptr + batch_idx * stride_has_initial_state
).to(tl.int1)
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
state_idx = tl.where(has_initial_state, state_idx, 0)
values = tl.load(
state_ptr + state_idx * stride_state_batch + offsets,
mask=mask & has_initial_state,
other=0.0,
)
tl.store(output_ptr + batch_idx * row_size + offsets, values, mask=mask)
def gather_initial_states(
state: torch.Tensor,
indices: torch.Tensor,
has_initial_state: torch.Tensor,
) -> torch.Tensor:
"""Gather dense state rows, replacing uninitialized rows with zeros."""
assert state.ndim >= 2
assert state.is_cuda or state.is_xpu
assert indices.ndim == 1 and has_initial_state.ndim == 1
assert indices.shape == has_initial_state.shape
assert indices.device == state.device
assert has_initial_state.device == state.device
assert indices.dtype in (torch.int32, torch.int64)
assert has_initial_state.dtype == torch.bool
row_size = state[0].numel()
# Mamba pages may pad stride(0), but each state row remains dense.
assert state[0].is_contiguous()
output = torch.empty(
(indices.numel(), *state.shape[1:]),
dtype=state.dtype,
device=state.device,
)
block_size = min(triton.next_power_of_2(row_size), 1024)
grid = (triton.cdiv(row_size, block_size), indices.numel())
_gather_initial_states_kernel[grid](
state,
indices,
has_initial_state,
output,
state.stride(0),
indices.stride(0),
has_initial_state.stride(0),
row_size=row_size,
BLOCK_SIZE=block_size,
num_warps=8,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return output
@@ -1,105 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
DispatchSpec,
TritonWarmupTensor,
triton_kernel_dispatcher_with_warmup,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@triton.jit
def _scatter_states_kernel(
state_ptr,
src_ptr,
indices_ptr,
stride_state_batch,
stride_src_batch,
stride_indices,
row_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
):
block_idx = tl.program_id(0)
batch_idx = tl.program_id(1)
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < row_size
if launch_pdl:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
values = tl.load(src_ptr + batch_idx * stride_src_batch + offsets, mask=mask)
tl.store(state_ptr + state_idx * stride_state_batch + offsets, values, mask=mask)
def _scatter_states_warmup_inputs(
*,
state_shape: tuple[int, ...],
state_dtype: torch.dtype,
indices_dtype: torch.dtype,
max_num_tokens: int,
) -> dict[str, object]:
num_tokens: Any = WarmupIntRange(1, max_num_tokens + 1)
return dict(
state=TritonWarmupTensor(
state_dtype,
shape=(1,) + state_shape,
),
src=TritonWarmupTensor(
state_dtype,
shape=(num_tokens,) + state_shape,
),
indices=TritonWarmupTensor(
indices_dtype,
shape=(num_tokens,),
),
)
@triton_kernel_dispatcher_with_warmup(
kernel=_scatter_states_kernel,
warmup_inputs=_scatter_states_warmup_inputs,
)
def scatter_states(
state: torch.Tensor,
src: torch.Tensor,
indices: torch.Tensor,
) -> DispatchSpec:
"""Scatter ``src`` rows into ``state`` at ``indices`` (in place).
Equivalent to ``state[indices] = src`` but non-atomic and bandwidth-bound,
since mamba cache slots are unique per sequence. ``gather_initial_states``
is the read-side counterpart.
"""
assert state.ndim >= 2
assert state.is_cuda
assert src.ndim == state.ndim
assert indices.ndim == 1
assert indices.device == state.device
assert src.shape[1:] == state.shape[1:]
assert src.shape[0] == indices.shape[0]
assert indices.dtype in (torch.int32, torch.int64)
row_size = state[0].numel()
assert state[0].is_contiguous()
assert src[0].is_contiguous()
block_size = min(triton.next_power_of_2(row_size), 1024)
grid = (triton.cdiv(row_size, block_size), indices.numel())
return grid, dict(
stride_state_batch=state.stride(0),
stride_src_batch=src.stride(0),
stride_indices=indices.stride(0),
row_size=row_size,
BLOCK_SIZE=block_size,
num_warps=8,
launch_pdl=current_platform.is_arch_support_pdl(),
)

Some files were not shown because too many files have changed in this diff Show More