Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
178 lines
5.9 KiB
Python
178 lines
5.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
|
|
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
|
|
# atom/mono/device/sync.py
|
|
"""The tagged mailbox: hand-offs between the CTAs of one GPU (device scope) and
|
|
between GPUs (system scope), every address a named region (``Addr``)."""
|
|
|
|
from dataclasses import dataclass
|
|
from enum import Enum
|
|
|
|
import flydsl.expr as fx
|
|
from aiter.ops.flydsl.kernels import buffer_ops as bo
|
|
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
|
|
from flydsl.expr.typing import T
|
|
|
|
from vllm.models.deepseek_v41.amd.mono.common.ops import (
|
|
CM_DEV,
|
|
CM_SYS,
|
|
bf16_pair,
|
|
rsrc,
|
|
traced,
|
|
)
|
|
|
|
POLL_MAX = 12 # mailbox specs polled per batch (bounds live registers)
|
|
|
|
|
|
@traced
|
|
def _spin(load_all, pending):
|
|
"""Re-issue ``load_all`` until ``pending`` clears; a side-effecting op in the
|
|
loop keeps the loads from being hoisted."""
|
|
v = load_all()
|
|
while pending(v):
|
|
rocdl.s_sleep(1)
|
|
v = load_all()
|
|
return v
|
|
|
|
|
|
class Space(Enum):
|
|
SCRATCH = "scratch" # this GPU's memory: device-scope hand-off between CTAs
|
|
PEER = "peer" # a peer-symmetric region: system-scope hand-off between GPUs
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Addr:
|
|
"""A mailbox region's address: the region's name and space, and the traced
|
|
address. The space sets the hand-off's scope: device for SCRATCH, system for
|
|
PEER."""
|
|
|
|
region: str
|
|
space: Space
|
|
value: object
|
|
|
|
|
|
def sreg(base, offset, region):
|
|
"""Scratch region ``region`` at byte ``offset`` of ``base``."""
|
|
return Addr(region, Space.SCRATCH, base + fx.Int64(offset))
|
|
|
|
|
|
def preg(base, offset, region):
|
|
"""Peer-symmetric region ``region`` at byte ``offset`` of a rank's buffer
|
|
``base``."""
|
|
return Addr(region, Space.PEER, base + fx.Int64(offset))
|
|
|
|
|
|
# the store / load cache policy of a hand-off in each space
|
|
_SCOPE_CM = {Space.SCRATCH: CM_DEV, Space.PEER: CM_SYS}
|
|
|
|
|
|
class Mailbox:
|
|
"""Tagged-pair hand-off between CTAs (device scope) and GPUs (system scope).
|
|
|
|
Every 32-bit value is stored next to this launch's tag; a consumer
|
|
polls the payload until all tags match, so a hand-off costs one round trip
|
|
(no store drain, no separate flag). The mono layer's launches tag their
|
|
hand-offs with their epoch (``layer.py``), so a pair from an earlier step
|
|
never carries a live tag. Every address is an ``Addr``: its space picks the
|
|
scope.
|
|
"""
|
|
|
|
def __init__(self, tag):
|
|
self.tag = tag
|
|
|
|
def put(self, a, i, v):
|
|
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
|
|
bo.buffer_store(
|
|
fx.Vector.from_elements([bits, self.tag], fx.Int32),
|
|
rsrc(a.value),
|
|
i * 2,
|
|
cache_modifier=_SCOPE_CM[a.space],
|
|
)
|
|
|
|
def put_bf(self, a, i, vs):
|
|
"""Elements i .. i+len(vs) (2 or 4, i aligned) as packed bf16 pairs, one
|
|
store."""
|
|
words = []
|
|
for j in range_constexpr(len(vs) // 2):
|
|
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
|
|
bo.buffer_store(
|
|
fx.Vector.from_elements(words, fx.Int32),
|
|
rsrc(a.value),
|
|
i,
|
|
cache_modifier=_SCOPE_CM[a.space],
|
|
)
|
|
|
|
def put_words(self, a, i, words):
|
|
"""Raw 32-bit words (1 or 2) at pairs i.. as (word, tag). Integer payloads
|
|
(fp8 words) must come this way: through an f32 value a NaN bit pattern is
|
|
not kept."""
|
|
vals = []
|
|
for w in words:
|
|
vals += [fx.Int32(w), self.tag]
|
|
bo.buffer_store(
|
|
fx.Vector.from_elements(vals, fx.Int32),
|
|
rsrc(a.value),
|
|
i * 2,
|
|
cache_modifier=_SCOPE_CM[a.space],
|
|
)
|
|
|
|
def poll(self, specs, batch=POLL_MAX):
|
|
"""Batched poll of pairs ``specs`` = [(Addr, pair index, npairs in {1, 2})],
|
|
all of one space.
|
|
|
|
All pairs are re-loaded together while any tag is stale, so a batch costs
|
|
one round trip after its last producer lands. Returns the value words."""
|
|
if const_expr(len(specs) == 0):
|
|
return []
|
|
if const_expr(len(specs) > batch):
|
|
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
|
|
spaces = {a.space for a, _, _ in specs}
|
|
assert len(spaces) == 1, f"one poll batch spans {spaces}"
|
|
cm = _SCOPE_CM[spaces.pop()]
|
|
bases = [a.value for a, _, _ in specs]
|
|
tag = self.tag
|
|
|
|
def load_all():
|
|
words = []
|
|
for b, (_, i, n) in zip(bases, specs):
|
|
w = fx.Vector(
|
|
bo.buffer_load(
|
|
rsrc(b),
|
|
fx.Int32(i) * 2,
|
|
vec_width=2 * n,
|
|
dtype=T.i32,
|
|
cache_modifier=cm,
|
|
)
|
|
)
|
|
words += [w[e] for e in range(2 * n)]
|
|
return fx.Vector.from_elements(words, fx.Int32)
|
|
|
|
nw = sum(2 * n for _, _, n in specs)
|
|
|
|
def pending(v):
|
|
bad = v[1] != tag
|
|
for e in range_constexpr(3, nw, 2):
|
|
bad = bad | (v[e] != tag)
|
|
return bad
|
|
|
|
v = _spin(load_all, pending)
|
|
outs, e = [], 0
|
|
for _, _, n in specs:
|
|
outs.append([v[e + 2 * q] for q in range(n)])
|
|
e += 2 * n
|
|
return outs
|
|
|
|
|
|
@traced
|
|
def publish(put, a, i, v, who):
|
|
"""This CTA's plain stores made readable by every CTA, then (thread ``who``)
|
|
the mailbox pair (``a``, i) := v. The stores are drained first: a flag may
|
|
not overtake the data it announces, and a workgroup-scope release fence does
|
|
not wait for them (LLVM's AMDGPU memory model omits vmcnt outside tgsplit
|
|
mode: the workgroup's waves share one CU, the readers here do not)."""
|
|
rocdl.s_waitcnt(vmcnt=0)
|
|
gpu.barrier()
|
|
if who:
|
|
put(a, i, v)
|