Files
Juntian LiuClaude Opus 5.5mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
efaca9547d [Perf][DSv4.1] Exchange packed FP8 Engram rows across DP and feed wkv MXFP8 (#59923)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2026-10-09 15:30:18 +08:00

174 lines
6.3 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Symmetric-memory Engram exchange for slots below ENGRAM_A2A_MIN_SLOT."""
from typing import TYPE_CHECKING
import torch
import torch.distributed as dist
from torch import nn
from vllm.distributed import get_engram_dp_group
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
if TYPE_CHECKING:
from vllm.models.deepseek_v41.common.engram import Engram
@triton.jit(do_not_specialize=["num_tokens"])
def _peer_lookup_kernel(
metadata,
id_ptrs,
output,
num_tokens,
HEADS: tl.constexpr,
LAYERS: tl.constexpr,
DP: tl.constexpr,
RANK: tl.constexpr,
DIM: tl.constexpr,
BLOCK: tl.constexpr,
):
"""Look up this rank's heads for every peer's tokens into packed rows."""
row = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
local_heads: tl.constexpr = HEADS // DP
valid = row < num_tokens * LAYERS * HEADS
token = row // (LAYERS * local_heads)
layer, head = row // local_heads % LAYERS, row % local_heads
ids = tl.load(id_ptrs + token // num_tokens, valid, other=0).to(
tl.pointer_type(tl.int32)
)
index = tl.load(
ids + (token % num_tokens * LAYERS + layer) * HEADS + RANK * local_heads + head,
valid,
other=-1,
).to(tl.int64)
# Per-layer metadata: weight pointer, scale pointer, vocab start, vocab end.
table = metadata + layer * 4
start, end = tl.load(table + 2), tl.load(table + 3)
owned = valid & (index >= start) & (index < end)
local = tl.where(owned, index - start, 0)
weight = tl.multiple_of(tl.load(table), DIM).to(tl.pointer_type(tl.uint8))
scales = tl.load(table + 1)
scales = tl.multiple_of(scales, DIM // 32).to(tl.pointer_type(tl.uint8))
col = tl.arange(0, DIM)
value = tl.load(
weight[:, None] + local[:, None] * DIM + col[None, :], owned[:, None], other=0
)
dst = row.to(tl.int64) * (DIM + DIM // 32)
tl.store(output + dst[:, None] + col[None, :], value, valid[:, None])
scale_cols = tl.arange(0, DIM // 32)
scale = tl.load(
scales[:, None] + local[:, None] * (DIM // 32) + scale_cols[None, :],
owned[:, None],
other=0,
)
tl.store(output + dst[:, None] + DIM + scale_cols[None, :], scale, valid[:, None])
class EngramPeerExchange(nn.Module):
"""Two phase barriers fence reuse of each other's buffers across forwards.
Pointer tables are buffers so level-2 sleep restores them on wake-up.
"""
def __init__(self, layers: list["Engram"], max_tokens: int):
import torch.distributed._symmetric_memory as symm_mem
super().__init__()
group = get_engram_dp_group()
assert group is not None
self.dp_size, self.dp_rank = group.world_size, group.rank_in_group
device = layers[0].staged_rows.device
self.num_layers = len(layers)
self.hash_start = layers[0].layer_hash_index
table = layers[0].embed_tokens
self.heads, self.dim = table.n_hash_cols, table.dim
self.ids = symm_mem.empty(
(max_tokens, self.num_layers, self.heads),
dtype=torch.int32,
device=device,
)
self.rows = symm_mem.empty(
(
max_tokens * self.dp_size,
self.num_layers,
self.heads // self.dp_size,
self.dim + self.dim // 32,
),
dtype=torch.uint8,
device=device,
)
self.id_handle = symm_mem.rendezvous(self.ids, group.device_group)
self.row_handle = symm_mem.rendezvous(self.rows, group.device_group)
metadata = []
for layer in layers:
table = layer.embed_tokens
weight, scale = table._storage()
metadata.append(
(
weight.data_ptr(),
scale.data_ptr(),
table.vocab_start_idx,
table.vocab_end_idx,
)
)
for name, data in (
("metadata", metadata),
("id_ptrs", self.id_handle.buffer_ptrs),
("row_ptrs", self.row_handle.buffer_ptrs),
):
self.register_buffer(
name,
torch.tensor(data, dtype=torch.int64, device=device),
persistent=False,
)
@classmethod
def create(
cls, layers: list["Engram"], max_tokens: int
) -> "EngramPeerExchange | None":
# The Engram DP group is node-local and every rank agrees on these.
group = get_engram_dp_group()
assert group is not None
if not (
current_platform.is_device_capability_family(100)
and group.world_size in (2, 4, 8)
and layers[0].embed_tokens.n_hash_cols % group.world_size == 0
):
return None
physical_id = current_platform.visible_device_id_to_physical_device_id(
torch.accelerator.current_device_index()
)
physical_ids = [None] * group.world_size
dist.all_gather_object(physical_ids, physical_id, group=group.cpu_group)
if not current_platform.is_fully_connected(physical_ids):
return None
return cls(layers, max_tokens)
def exchange(self, hashes: torch.Tensor, slot: int) -> torch.Tensor:
"""Look up this rank's heads for every peer; return per-rank row pointers."""
tokens = hashes.shape[0]
assert 0 < slot <= self.ids.shape[0] and tokens <= slot
hash_end = self.hash_start + self.num_layers
self.ids[:tokens].copy_(hashes[:, self.hash_start : hash_end])
if tokens < slot:
self.ids[tokens:slot].fill_(-1)
# The preceding rows barrier completed every peer's hash reads. This
# barrier completes their previous output reads before rows are reused.
self.id_handle.barrier()
_peer_lookup_kernel[(triton.cdiv(slot * self.num_layers * self.heads, 16),)](
self.metadata,
self.id_ptrs,
self.rows,
slot,
HEADS=self.heads,
LAYERS=self.num_layers,
DP=self.dp_size,
RANK=self.dp_rank,
DIM=self.dim,
BLOCK=16,
)
self.row_handle.barrier()
return self.row_ptrs