[AuxOutput] Add block-keyed storage for routed-expert outputs (#45635)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
Roger Wang
Nick Hill
Codex
parent
0aee727ff6
commit
0b7f11a1ee
@@ -10,15 +10,12 @@ from collections.abc import Callable
|
||||
from functools import partial
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec
|
||||
from vllm.v1.outputs import RoutedExpertsTensors
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -46,37 +43,29 @@ def _get_routed_experts_shape(vllm_config: VllmConfig) -> tuple[int, int, int]:
|
||||
class RoutedExpertsCapturer:
|
||||
"""Worker-side capturer for routed experts, lives on GPU.
|
||||
|
||||
Layer-level hooks call :meth:`capture` from inside the forward pass
|
||||
with the per-layer ``topk_ids`` tensor. The tensor is sliced to the
|
||||
tokens owned by this DP rank and written into a preallocated device
|
||||
buffer. At the end of the step, :class:`GPUModelRunner` reads the
|
||||
device buffer, issues a D2H copy into a pinned CPU buffer, and hands
|
||||
the result to the scheduler via :class:`RoutedExpertsLists`.
|
||||
Layer-level hooks call :meth:`capture` inside the forward pass. Routing
|
||||
rows owned by this DP rank are written into a preallocated device buffer.
|
||||
|
||||
The device / pinned-CPU transit buffers use ``torch.int32`` (not a
|
||||
narrow ``uint8``/``uint16`` sized by ``num_experts``). This keeps the
|
||||
SP all-gather path free of dtype casts, matches the router's native
|
||||
``topk_ids`` indices dtype more closely, and costs only a few MB per
|
||||
worker (``max_num_batched_tokens * num_layers * top_k * 4`` bytes).
|
||||
The scheduler-side slot buffer
|
||||
(``RoutedExpertsManager.routed_experts_by_slot``) still uses the
|
||||
narrow dtype -- numpy fancy-index assignment in ``store_batch``
|
||||
narrows the data on the way in.
|
||||
The device buffer uses ``int32``. Stable snapshots use the narrowest dtype
|
||||
that can represent every logical expert ID.
|
||||
|
||||
Invariants:
|
||||
- One instance per worker; shape is fixed at init and covers the
|
||||
worst-case step (``max_num_batched_tokens`` tokens).
|
||||
- Every routed layer overwrites the current step's token rows.
|
||||
- ``device_buffer.dtype`` is ``torch.int32``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_num_batched_tokens: int,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
num_layers, _, num_experts_per_tok = _get_routed_experts_shape(vllm_config)
|
||||
num_layers, num_experts, num_experts_per_tok = _get_routed_experts_shape(
|
||||
vllm_config
|
||||
)
|
||||
self.shape_per_token = (num_layers, num_experts_per_tok)
|
||||
self.output_dtype_name = "uint8" if num_experts <= 256 else "uint16"
|
||||
self.output_dtype = getattr(torch, self.output_dtype_name)
|
||||
logger.info(
|
||||
"RoutedExpertsCapturer: allocating buffer with "
|
||||
"max_tokens=%d, num_layers=%d, num_experts_per_tok=%d "
|
||||
@@ -87,23 +76,12 @@ class RoutedExpertsCapturer:
|
||||
vllm_config.model_config.hf_text_config.model_type,
|
||||
)
|
||||
self.device_buffer = torch.zeros(
|
||||
(
|
||||
max_num_batched_tokens,
|
||||
num_layers,
|
||||
num_experts_per_tok,
|
||||
),
|
||||
# Use int32 for the device / host transit buffers: it
|
||||
# matches the router's native topk_ids dtype, is universally
|
||||
# supported by NCCL (uint8/uint16 are version-dependent),
|
||||
# and the extra bytes are small (few MB per worker). The
|
||||
# big scheduler-side slot buffer stays narrow.
|
||||
(max_num_batched_tokens, *self.shape_per_token),
|
||||
dtype=torch.int32,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
# KV cache group whose slot layout the routing data is keyed by.
|
||||
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
|
||||
|
||||
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
|
||||
"""Capture expert routing decisions for a specific layer.
|
||||
@@ -139,31 +117,21 @@ class RoutedExpertsCapturer:
|
||||
|
||||
"""
|
||||
ctx = get_forward_context()
|
||||
if ctx.dp_metadata is None: # single dp
|
||||
start_loc = 0
|
||||
end_loc = topk_ids.shape[0]
|
||||
token_num_per_dp = topk_ids.shape[0]
|
||||
else: # multi dp
|
||||
if ctx.dp_metadata is None:
|
||||
local_topk_ids = topk_ids
|
||||
else:
|
||||
num_tokens_dp = ctx.dp_metadata.num_tokens_across_dp_cpu
|
||||
token_num_per_dp = int(num_tokens_dp[self.dp_rank].item())
|
||||
num_local_tokens = int(num_tokens_dp[self.dp_rank].item())
|
||||
total = int(num_tokens_dp.sum().item())
|
||||
n = topk_ids.shape[0]
|
||||
shard_sizes = getattr(ctx.dp_metadata, "local_sizes", None)
|
||||
gathered_size = sum(shard_sizes) if shard_sizes is not None else -1
|
||||
|
||||
if n == total:
|
||||
# Naive dispatch: all DP ranks' tokens concatenated
|
||||
# before routing. This rank owns tokens
|
||||
# [end_loc - token_num_per_dp, end_loc).
|
||||
cumsum = torch.cumsum(num_tokens_dp, dim=0)
|
||||
end_loc = int(cumsum[self.dp_rank].item())
|
||||
start_loc = end_loc - token_num_per_dp
|
||||
elif n == token_num_per_dp:
|
||||
# Modular-kernel path: DP combine happens inside
|
||||
# quant_method.apply; select_experts only sees this
|
||||
# rank's tokens, take the whole tensor.
|
||||
start_loc = 0
|
||||
end_loc = token_num_per_dp
|
||||
end = int(num_tokens_dp[: self.dp_rank + 1].sum().item())
|
||||
local_topk_ids = topk_ids[end - num_local_tokens : end]
|
||||
elif n == num_local_tokens:
|
||||
local_topk_ids = topk_ids
|
||||
elif shard_sizes is not None and n == gathered_size:
|
||||
# Naive DP+EP dispatch gathers the sequence-parallel shards in
|
||||
# flattened DP-then-TP order. ``local_sizes`` is the exact
|
||||
@@ -173,12 +141,11 @@ class RoutedExpertsCapturer:
|
||||
assert len(shard_sizes) % num_dp_ranks == 0
|
||||
shards_per_dp_rank = len(shard_sizes) // num_dp_ranks
|
||||
first_shard = self.dp_rank * shards_per_dp_rank
|
||||
start_loc = sum(shard_sizes[:first_shard])
|
||||
end_loc = start_loc + token_num_per_dp
|
||||
start = sum(shard_sizes[:first_shard])
|
||||
local_topk_ids = topk_ids[start : start + num_local_tokens]
|
||||
elif (
|
||||
self.tp_size > 1
|
||||
and n != token_num_per_dp
|
||||
and n == (token_num_per_dp + self.tp_size - 1) // self.tp_size
|
||||
and n == (num_local_tokens + self.tp_size - 1) // self.tp_size
|
||||
):
|
||||
# SP + modular-kernel path. All-gather across the TP
|
||||
# group along dim=0 to reconstruct the full per-DP-rank
|
||||
@@ -193,60 +160,30 @@ class RoutedExpertsCapturer:
|
||||
#
|
||||
# ``topk_ids`` is already whatever the router produced
|
||||
# (typically int32/int64, both supported by NCCL); the
|
||||
# downstream ``device_buffer[...] = topk_ids[...]``
|
||||
# setitem narrows into int32 automatically.
|
||||
topk_ids = get_tp_group().all_gather(topk_ids, dim=0)
|
||||
start_loc = 0
|
||||
end_loc = token_num_per_dp
|
||||
# downstream buffer assignment narrows to the capture dtype.
|
||||
local_topk_ids = get_tp_group().all_gather(topk_ids, dim=0)[
|
||||
:num_local_tokens
|
||||
]
|
||||
else:
|
||||
sp_expected = (
|
||||
(token_num_per_dp + self.tp_size - 1) // self.tp_size
|
||||
if self.tp_size > 0
|
||||
else -1
|
||||
)
|
||||
sp_expected = (num_local_tokens + self.tp_size - 1) // self.tp_size
|
||||
raise AssertionError(
|
||||
"RoutedExpertsCapturer: unexpected topk_ids batch "
|
||||
f"dim {n} (expected {total}, {token_num_per_dp}, "
|
||||
f"dim {n} (expected {total}, {num_local_tokens}, "
|
||||
f"{gathered_size}, or {sp_expected} for "
|
||||
f"dp_rank={self.dp_rank}, tp_size={self.tp_size})"
|
||||
)
|
||||
|
||||
if layer_id >= self.device_buffer.shape[1]:
|
||||
if not 0 <= layer_id < self.device_buffer.shape[1]:
|
||||
raise IndexError(
|
||||
f"routed-experts layer {layer_id} exceeds capture buffer "
|
||||
f"layer count {self.device_buffer.shape[1]}"
|
||||
)
|
||||
|
||||
self.device_buffer[:token_num_per_dp, layer_id, :] = topk_ids[
|
||||
start_loc:end_loc, :
|
||||
]
|
||||
self.device_buffer[: len(local_topk_ids), layer_id] = local_topk_ids
|
||||
|
||||
def get_device_buffer(self) -> torch.Tensor:
|
||||
"""Return the underlying device buffer so the model runner can
|
||||
issue the D2H copy. The tensor is shared; callers must either
|
||||
clone or fully drain it before the next forward pass overwrites it.
|
||||
"""
|
||||
return self.device_buffer
|
||||
|
||||
def get_routed_experts(
|
||||
self, slot_mappings: torch.Tensor, num_tokens: int
|
||||
) -> RoutedExpertsTensors:
|
||||
"""Snapshot this step's routing data and its attention slot mapping.
|
||||
|
||||
Both tensors are cloned since the capture buffer and the shared
|
||||
``slot_mappings`` are overwritten by the next step, which may race
|
||||
with an in-flight D2H copy.
|
||||
|
||||
Args:
|
||||
slot_mappings: Per-KV-cache-group slot mappings for this step,
|
||||
shape ``(num_kv_cache_groups, max_num_batched_tokens)``.
|
||||
num_tokens: Total number of tokens scheduled in this step.
|
||||
|
||||
"""
|
||||
return RoutedExpertsTensors(
|
||||
routing_data=self.device_buffer[:num_tokens].clone(),
|
||||
slot_mapping=slot_mappings[self.attn_gid, :num_tokens].clone(),
|
||||
)
|
||||
def snapshot_routing_data(self, num_tokens: int) -> torch.Tensor:
|
||||
"""Return a stable snapshot of the current routing data."""
|
||||
return self.device_buffer[:num_tokens].to(self.output_dtype)
|
||||
|
||||
|
||||
def bind_routed_experts_capturer(
|
||||
@@ -268,20 +205,12 @@ def bind_routed_experts_capturer(
|
||||
continue
|
||||
if not isinstance(module, MoERunner):
|
||||
continue
|
||||
layer_id = module.layer_id
|
||||
|
||||
def capture_fn(
|
||||
topk_ids: torch.Tensor,
|
||||
layer_id: int = layer_id,
|
||||
capturer: RoutedExpertsCapturer = capturer,
|
||||
) -> None:
|
||||
capturer.capture(layer_id, topk_ids)
|
||||
|
||||
capture_fn = partial(capturer.capture, module.layer_id)
|
||||
quant_method = module._quant_method
|
||||
moe_kernel = getattr(quant_method, "moe_kernel", None)
|
||||
impl = getattr(moe_kernel, "impl", None)
|
||||
fused_experts = getattr(impl, "fused_experts", None)
|
||||
if quant_method.is_monolithic:
|
||||
moe_kernel = getattr(quant_method, "moe_kernel", None)
|
||||
impl = getattr(moe_kernel, "impl", None)
|
||||
fused_experts = getattr(impl, "fused_experts", None)
|
||||
if not (
|
||||
isinstance(fused_experts, FusedMoEExpertsMonolithic)
|
||||
and fused_experts.supports_routing_replay_capture()
|
||||
@@ -303,132 +232,3 @@ def bind_routed_experts_capturer(
|
||||
|
||||
if num_bound == 0:
|
||||
raise ValueError("No supported MoE router found for routed-experts capture.")
|
||||
|
||||
|
||||
def get_routed_experts_attn_gid(kv_cache_config: KVCacheConfig) -> int:
|
||||
"""Return the full-attention KV cache group used for routed experts."""
|
||||
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if is_full_attention_spec(group.kv_cache_spec):
|
||||
return gid
|
||||
raise ValueError("Routed-experts capture requires a full-attention KV cache group.")
|
||||
|
||||
|
||||
class RoutedExpertsManager:
|
||||
"""Scheduler-side slot-indexed buffer for routed experts.
|
||||
|
||||
Lives on CPU in the scheduler process. Each slot corresponds to
|
||||
``block_id * block_size + offset_in_block`` where ``block_id`` is
|
||||
drawn from the physical KV-cache block pool, so routing data is
|
||||
tied to physical blocks and naturally survives preemption for
|
||||
prefix-cached blocks (prefix hits re-expose the same slots).
|
||||
|
||||
Data flow per step:
|
||||
1. Worker D2Hs its device capture buffer into
|
||||
:class:`RoutedExpertsLists` and returns it via
|
||||
:class:`ModelRunnerOutput`.
|
||||
2. Scheduler calls :meth:`store_batch` with that step's
|
||||
``(routing_data, slot_mapping)`` — a single CPU->CPU
|
||||
fancy-index assign, ~few MB per step.
|
||||
3. On request completion / abort / preemption, the scheduler
|
||||
calls :meth:`get` with the request's block IDs to recover
|
||||
the full per-token routing.
|
||||
|
||||
Memory: ``routed_experts_by_slot`` is sized for the whole block
|
||||
pool (``num_blocks * block_size`` slots). For large block pools
|
||||
this can reach multiple GB; see the init log for the exact size.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
# Hybrid models also have KV groups whose slot layout differs.
|
||||
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
|
||||
attn_group = kv_cache_config.kv_cache_groups[self.attn_gid]
|
||||
self.block_size = attn_group.kv_cache_spec.block_size
|
||||
|
||||
# All kv_cache_groups share the same physical block pool, so
|
||||
# block IDs span [0, num_blocks) regardless of how many groups
|
||||
# exist. Sizing to the full pool avoids index-out-of-range
|
||||
# when different groups happen to land on the same block.
|
||||
num_layers, num_experts, num_experts_per_tok = _get_routed_experts_shape(
|
||||
vllm_config
|
||||
)
|
||||
max_num_slots = kv_cache_config.num_blocks * self.block_size
|
||||
# Expert IDs are 0..num_experts-1; uint8 fits 256 distinct
|
||||
# values so the boundary is ``<= 256`` (NOT ``< 256``). Keeping
|
||||
# this narrow matters because the slot buffer is sized for the
|
||||
# whole block pool and can reach multiple GB.
|
||||
expert_id_dtype = np.uint8 if num_experts <= 256 else np.uint16
|
||||
self.routed_experts_by_slot = np.zeros(
|
||||
(
|
||||
max_num_slots,
|
||||
num_layers,
|
||||
num_experts_per_tok,
|
||||
),
|
||||
dtype=expert_id_dtype,
|
||||
)
|
||||
logger.info(
|
||||
"RoutedExpertsManager CPU buffer: %.2f GB "
|
||||
"(slots=%d, layers=%d, top_k=%d, dtype=%s)",
|
||||
self.routed_experts_by_slot.nbytes / 1e9,
|
||||
max_num_slots,
|
||||
num_layers,
|
||||
num_experts_per_tok,
|
||||
self.routed_experts_by_slot.dtype.name,
|
||||
)
|
||||
|
||||
def store_batch(self, data: np.ndarray, slot_mapping: np.ndarray) -> None:
|
||||
"""Persist one step's routed experts into the slot buffer.
|
||||
|
||||
Equivalent to ``slot_buffer[slot_mapping] = data``; numpy fancy
|
||||
indexing handles repeated / out-of-order indices. Called once
|
||||
per scheduler step in ``update_from_output``.
|
||||
"""
|
||||
self.routed_experts_by_slot[slot_mapping] = data
|
||||
|
||||
def get(
|
||||
self,
|
||||
block_ids: list[int],
|
||||
num_tokens: int,
|
||||
token_start: int = 0,
|
||||
) -> np.ndarray:
|
||||
"""Read routed experts data for a completed / preempted request.
|
||||
|
||||
Reconstructs a per-token slot_mapping from the request's block
|
||||
IDs and returns the routing slice. Because numpy fancy indexing
|
||||
returns a **copy** (not a view), the returned ndarray is safe
|
||||
to hold across subsequent :meth:`store_batch` calls — do not
|
||||
replace the fancy index with a slice without re-verifying.
|
||||
|
||||
Args:
|
||||
block_ids: Block IDs from the attention KV-cache group.
|
||||
num_tokens: Number of tokens that have gone through a forward
|
||||
pass and therefore have routing data written to their
|
||||
slots (typically ``request.num_tokens - 1``; the last
|
||||
sampled token has not been forwarded yet). Slots beyond
|
||||
``request.num_computed_tokens`` are zero-initialized.
|
||||
token_start: Skip the first ``token_start`` tokens from the
|
||||
result. The slot_mapping is sliced before the fancy-index
|
||||
read, so only the requested slots are fetched — no large
|
||||
intermediate array is allocated. Clamped to
|
||||
``[0, num_tokens]`` automatically.
|
||||
|
||||
Returns:
|
||||
Array of shape (num_tokens - token_start, num_layers,
|
||||
num_experts_per_tok).
|
||||
|
||||
"""
|
||||
bs = self.block_size
|
||||
block_ids_array = np.array(block_ids, dtype=np.int32)
|
||||
block_offsets = np.arange(bs)
|
||||
# slot = block_id * block_size + offset_in_block; flatten the
|
||||
# (num_blocks, block_size) grid and trim to num_tokens, then
|
||||
# skip the first token_start entries so only the requested
|
||||
# range is fetched in a single fancy-index read.
|
||||
slot_mapping = (
|
||||
block_ids_array.reshape(-1, 1) * bs + block_offsets.reshape(1, -1)
|
||||
).flatten()[:num_tokens]
|
||||
slot_mapping = slot_mapping[token_start:]
|
||||
return self.routed_experts_by_slot[slot_mapping]
|
||||
|
||||
Reference in New Issue
Block a user