efaca9547d
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
174 lines
6.3 KiB
Python
174 lines
6.3 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Symmetric-memory Engram exchange for slots below ENGRAM_A2A_MIN_SLOT."""
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
import torch
|
|
import torch.distributed as dist
|
|
from torch import nn
|
|
|
|
from vllm.distributed import get_engram_dp_group
|
|
from vllm.platforms import current_platform
|
|
from vllm.triton_utils import tl, triton
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.models.deepseek_v41.common.engram import Engram
|
|
|
|
|
|
@triton.jit(do_not_specialize=["num_tokens"])
|
|
def _peer_lookup_kernel(
|
|
metadata,
|
|
id_ptrs,
|
|
output,
|
|
num_tokens,
|
|
HEADS: tl.constexpr,
|
|
LAYERS: tl.constexpr,
|
|
DP: tl.constexpr,
|
|
RANK: tl.constexpr,
|
|
DIM: tl.constexpr,
|
|
BLOCK: tl.constexpr,
|
|
):
|
|
"""Look up this rank's heads for every peer's tokens into packed rows."""
|
|
row = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
|
local_heads: tl.constexpr = HEADS // DP
|
|
valid = row < num_tokens * LAYERS * HEADS
|
|
token = row // (LAYERS * local_heads)
|
|
layer, head = row // local_heads % LAYERS, row % local_heads
|
|
ids = tl.load(id_ptrs + token // num_tokens, valid, other=0).to(
|
|
tl.pointer_type(tl.int32)
|
|
)
|
|
index = tl.load(
|
|
ids + (token % num_tokens * LAYERS + layer) * HEADS + RANK * local_heads + head,
|
|
valid,
|
|
other=-1,
|
|
).to(tl.int64)
|
|
# Per-layer metadata: weight pointer, scale pointer, vocab start, vocab end.
|
|
table = metadata + layer * 4
|
|
start, end = tl.load(table + 2), tl.load(table + 3)
|
|
owned = valid & (index >= start) & (index < end)
|
|
local = tl.where(owned, index - start, 0)
|
|
weight = tl.multiple_of(tl.load(table), DIM).to(tl.pointer_type(tl.uint8))
|
|
scales = tl.load(table + 1)
|
|
scales = tl.multiple_of(scales, DIM // 32).to(tl.pointer_type(tl.uint8))
|
|
col = tl.arange(0, DIM)
|
|
value = tl.load(
|
|
weight[:, None] + local[:, None] * DIM + col[None, :], owned[:, None], other=0
|
|
)
|
|
dst = row.to(tl.int64) * (DIM + DIM // 32)
|
|
tl.store(output + dst[:, None] + col[None, :], value, valid[:, None])
|
|
scale_cols = tl.arange(0, DIM // 32)
|
|
scale = tl.load(
|
|
scales[:, None] + local[:, None] * (DIM // 32) + scale_cols[None, :],
|
|
owned[:, None],
|
|
other=0,
|
|
)
|
|
tl.store(output + dst[:, None] + DIM + scale_cols[None, :], scale, valid[:, None])
|
|
|
|
|
|
class EngramPeerExchange(nn.Module):
|
|
"""Two phase barriers fence reuse of each other's buffers across forwards.
|
|
|
|
Pointer tables are buffers so level-2 sleep restores them on wake-up.
|
|
"""
|
|
|
|
def __init__(self, layers: list["Engram"], max_tokens: int):
|
|
import torch.distributed._symmetric_memory as symm_mem
|
|
|
|
super().__init__()
|
|
group = get_engram_dp_group()
|
|
assert group is not None
|
|
self.dp_size, self.dp_rank = group.world_size, group.rank_in_group
|
|
device = layers[0].staged_rows.device
|
|
self.num_layers = len(layers)
|
|
self.hash_start = layers[0].layer_hash_index
|
|
table = layers[0].embed_tokens
|
|
self.heads, self.dim = table.n_hash_cols, table.dim
|
|
self.ids = symm_mem.empty(
|
|
(max_tokens, self.num_layers, self.heads),
|
|
dtype=torch.int32,
|
|
device=device,
|
|
)
|
|
self.rows = symm_mem.empty(
|
|
(
|
|
max_tokens * self.dp_size,
|
|
self.num_layers,
|
|
self.heads // self.dp_size,
|
|
self.dim + self.dim // 32,
|
|
),
|
|
dtype=torch.uint8,
|
|
device=device,
|
|
)
|
|
self.id_handle = symm_mem.rendezvous(self.ids, group.device_group)
|
|
self.row_handle = symm_mem.rendezvous(self.rows, group.device_group)
|
|
metadata = []
|
|
for layer in layers:
|
|
table = layer.embed_tokens
|
|
weight, scale = table._storage()
|
|
metadata.append(
|
|
(
|
|
weight.data_ptr(),
|
|
scale.data_ptr(),
|
|
table.vocab_start_idx,
|
|
table.vocab_end_idx,
|
|
)
|
|
)
|
|
for name, data in (
|
|
("metadata", metadata),
|
|
("id_ptrs", self.id_handle.buffer_ptrs),
|
|
("row_ptrs", self.row_handle.buffer_ptrs),
|
|
):
|
|
self.register_buffer(
|
|
name,
|
|
torch.tensor(data, dtype=torch.int64, device=device),
|
|
persistent=False,
|
|
)
|
|
|
|
@classmethod
|
|
def create(
|
|
cls, layers: list["Engram"], max_tokens: int
|
|
) -> "EngramPeerExchange | None":
|
|
# The Engram DP group is node-local and every rank agrees on these.
|
|
group = get_engram_dp_group()
|
|
assert group is not None
|
|
if not (
|
|
current_platform.is_device_capability_family(100)
|
|
and group.world_size in (2, 4, 8)
|
|
and layers[0].embed_tokens.n_hash_cols % group.world_size == 0
|
|
):
|
|
return None
|
|
physical_id = current_platform.visible_device_id_to_physical_device_id(
|
|
torch.accelerator.current_device_index()
|
|
)
|
|
physical_ids = [None] * group.world_size
|
|
dist.all_gather_object(physical_ids, physical_id, group=group.cpu_group)
|
|
if not current_platform.is_fully_connected(physical_ids):
|
|
return None
|
|
return cls(layers, max_tokens)
|
|
|
|
def exchange(self, hashes: torch.Tensor, slot: int) -> torch.Tensor:
|
|
"""Look up this rank's heads for every peer; return per-rank row pointers."""
|
|
tokens = hashes.shape[0]
|
|
assert 0 < slot <= self.ids.shape[0] and tokens <= slot
|
|
hash_end = self.hash_start + self.num_layers
|
|
self.ids[:tokens].copy_(hashes[:, self.hash_start : hash_end])
|
|
if tokens < slot:
|
|
self.ids[tokens:slot].fill_(-1)
|
|
# The preceding rows barrier completed every peer's hash reads. This
|
|
# barrier completes their previous output reads before rows are reused.
|
|
self.id_handle.barrier()
|
|
_peer_lookup_kernel[(triton.cdiv(slot * self.num_layers * self.heads, 16),)](
|
|
self.metadata,
|
|
self.id_ptrs,
|
|
self.rows,
|
|
slot,
|
|
HEADS=self.heads,
|
|
LAYERS=self.num_layers,
|
|
DP=self.dp_size,
|
|
RANK=self.dp_rank,
|
|
DIM=self.dim,
|
|
BLOCK=16,
|
|
)
|
|
self.row_handle.barrier()
|
|
return self.row_ptrs
|