Files
0b7f11a1ee [AuxOutput] Add block-keyed storage for routed-expert outputs (#45635)
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
2026-09-21 10:25:05 +00:00

220 lines
8.0 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routed-experts auxiliary output keys, publication, and materialization."""
from __future__ import annotations
from collections.abc import Hashable, Iterable, Sequence
from dataclasses import dataclass
from typing import Any
import numpy as np
from vllm.distributed.aux_output_connector.store import (
BackgroundBlockObjectStore,
BlockObject,
BlockObjectStore,
)
@dataclass
class _RequestTail:
slot: int
block_start: int
length: int = 0
class RoutedExpertsBuffer:
"""Own incomplete and unkeyed full routed-experts blocks."""
def __init__(
self,
dtype: np.dtype[Any],
shape_per_token: tuple[int, ...],
block_size: int,
max_num_seqs: int,
max_num_batched_tokens: int,
max_concurrent_batches: int,
) -> None:
self.dtype = dtype
self.shape_per_token = shape_per_token
self.block_size = block_size
max_step_blocks = (
max_num_batched_tokens + block_size - 1
) // block_size + max_num_seqs
# In-flight full blocks plus one incomplete tail per active request.
max_blocks = max_concurrent_batches * max_step_blocks + max_num_seqs
self._rows: np.ndarray = np.empty(
(max_blocks, block_size, *shape_per_token), dtype=dtype
)
self._free_slots = list(range(len(self._rows) - 1, -1, -1))
self._owned_slots: dict[int, int] = {}
self._requests: dict[Hashable, _RequestTail] = {}
def _tail(self, request_id: Hashable, block_start: int) -> _RequestTail:
tail = self._requests.get(request_id)
if tail is not None:
assert tail.block_start == block_start, (
"auxiliary output capture skipped an incomplete block: "
f"request={request_id}, expected={tail.block_start}, "
f"actual={block_start}"
)
return tail
assert self._free_slots, "auxiliary output block pool is exhausted"
tail = _RequestTail(self._free_slots.pop(), block_start)
self._requests[request_id] = tail
return tail
def capture(
self, request_id: Hashable, token_start: int, rows: np.ndarray
) -> list[tuple[int, np.ndarray]]:
"""Stage rows and return completed blocks without retaining them."""
rows = np.asarray(rows)
assert rows.shape[1:] == self.shape_per_token and rows.dtype == self.dtype, (
"routed-experts capture profile changed"
)
if token_start < 0:
raise ValueError("auxiliary output token start must be non-negative")
completed: list[tuple[int, np.ndarray]] = []
offset = 0
while offset < len(rows):
position = token_start + offset
block_start = position // self.block_size * self.block_size
local_start = position - block_start
# Full aligned input blocks do not need tail staging.
if (
request_id not in self._requests
and local_start == 0
and len(rows) - offset >= self.block_size
):
completed.append((block_start, rows[offset : offset + self.block_size]))
offset += self.block_size
continue
tail = self._tail(request_id, block_start)
assert local_start == tail.length, (
"auxiliary output capture is not contiguous: "
f"request={request_id}, expected={block_start + tail.length}, "
f"actual={position}"
)
count = min(self.block_size - local_start, len(rows) - offset)
if count == 1:
self._rows[tail.slot, local_start] = rows[offset]
else:
self._rows[tail.slot, local_start : local_start + count] = rows[
offset : offset + count
]
tail.length += count
offset += count
if tail.length == self.block_size:
block = self._rows[tail.slot]
del self._requests[request_id]
self._owned_slots[id(block)] = tail.slot
completed.append((block_start, block))
return completed
def read(
self, request_id: Hashable, token_start: int, token_end: int
) -> np.ndarray:
tail = self._requests.get(request_id)
assert tail is not None, (
f"auxiliary output buffer is missing request {request_id}"
)
local_start = token_start - tail.block_start
local_end = token_end - tail.block_start
assert 0 <= local_start < local_end <= tail.length, (
"auxiliary output range is unavailable: "
f"request={request_id}, range=[{token_start}, {token_end}), "
f"available=[{tail.block_start}, {tail.block_start + tail.length})"
)
return self._rows[tail.slot, local_start:local_end].copy()
def retain_block(self, rows: np.ndarray) -> np.ndarray:
"""Retain one unkeyed block after the current capture call."""
if id(rows) in self._owned_slots:
return rows
assert self._free_slots, "auxiliary output block pool is exhausted"
slot = self._free_slots.pop()
retained = self._rows[slot]
retained[...] = rows
self._owned_slots[id(retained)] = slot
return retained
def release_block(self, rows: np.ndarray) -> None:
slot = self._owned_slots.pop(id(rows), None)
if slot is not None:
self._free_slots.append(slot)
def discard(self, request_id: Hashable) -> None:
tail = self._requests.pop(request_id, None)
if tail is not None:
self._free_slots.append(tail.slot)
def reset(self) -> None:
self._requests.clear()
self._owned_slots.clear()
self._free_slots[:] = range(len(self._rows) - 1, -1, -1)
def routed_experts_keys(
block_hashes: Iterable[bytes], aux_output_namespace: str
) -> list[str]:
prefix = f"vllm-artifact/{aux_output_namespace}/"
return [prefix + block_hash.hex() for block_hash in block_hashes]
def materialize_routed_experts(
store: BackgroundBlockObjectStore | BlockObjectStore,
aux_output_keys: list[str],
*,
shape_per_token: tuple[int, ...],
dtype: np.dtype[Any],
) -> np.ndarray:
if not aux_output_keys:
raise ValueError("routed-experts auxiliary output key list must not be empty")
payload = store.get_concatenated(aux_output_keys)
return np.frombuffer(payload, dtype=dtype).reshape((-1, *shape_per_token))
def publish_routed_experts(
store: BackgroundBlockObjectStore | BlockObjectStore,
*,
batches: Sequence[tuple[Sequence[str], list[tuple[int, np.ndarray]]]],
block_size: int,
retain_keys: Sequence[str] = (),
release_keys: Sequence[str] = (),
) -> None:
"""Publish immutable full R3 blocks."""
objects = []
for aux_output_keys, blocks in batches:
for block_start, array in blocks:
if block_start < 0 or block_start % block_size:
raise ValueError(
"auxiliary output block start is not hash-block aligned"
)
block_index = block_start // block_size
if block_index >= len(aux_output_keys):
raise ValueError(
"auxiliary output block has no corresponding key: "
f"start={block_start}, index={block_index}, "
f"keys={len(aux_output_keys)}, block_size={block_size}"
)
if len(array) != block_size:
raise ValueError(
"auxiliary output block length does not match hash block size"
)
objects.append(
BlockObject(
key=aux_output_keys[block_index],
payload=array.tobytes(order="C"),
)
)
store.put(
objects,
retain_keys=retain_keys,
release_keys=release_keys,
)