Files

178 lines
5.9 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from ROCm/ATOM (https://github.com/ROCm/ATOM) at 97359d6, MIT License,
# Copyright (C) 2026, Advanced Micro Devices, Inc.:
# atom/mono/device/sync.py
"""The tagged mailbox: hand-offs between the CTAs of one GPU (device scope) and
between GPUs (system scope), every address a named region (``Addr``)."""
from dataclasses import dataclass
from enum import Enum
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import buffer_ops as bo
from flydsl.expr import const_expr, gpu, range_constexpr, rocdl
from flydsl.expr.typing import T
from vllm.models.deepseek_v41.amd.mono.common.ops import (
CM_DEV,
CM_SYS,
bf16_pair,
rsrc,
traced,
)
POLL_MAX = 12 # mailbox specs polled per batch (bounds live registers)
@traced
def _spin(load_all, pending):
"""Re-issue ``load_all`` until ``pending`` clears; a side-effecting op in the
loop keeps the loads from being hoisted."""
v = load_all()
while pending(v):
rocdl.s_sleep(1)
v = load_all()
return v
class Space(Enum):
SCRATCH = "scratch" # this GPU's memory: device-scope hand-off between CTAs
PEER = "peer" # a peer-symmetric region: system-scope hand-off between GPUs
@dataclass(frozen=True)
class Addr:
"""A mailbox region's address: the region's name and space, and the traced
address. The space sets the hand-off's scope: device for SCRATCH, system for
PEER."""
region: str
space: Space
value: object
def sreg(base, offset, region):
"""Scratch region ``region`` at byte ``offset`` of ``base``."""
return Addr(region, Space.SCRATCH, base + fx.Int64(offset))
def preg(base, offset, region):
"""Peer-symmetric region ``region`` at byte ``offset`` of a rank's buffer
``base``."""
return Addr(region, Space.PEER, base + fx.Int64(offset))
# the store / load cache policy of a hand-off in each space
_SCOPE_CM = {Space.SCRATCH: CM_DEV, Space.PEER: CM_SYS}
class Mailbox:
"""Tagged-pair hand-off between CTAs (device scope) and GPUs (system scope).
Every 32-bit value is stored next to this launch's tag; a consumer
polls the payload until all tags match, so a hand-off costs one round trip
(no store drain, no separate flag). The mono layer's launches tag their
hand-offs with their epoch (``layer.py``), so a pair from an earlier step
never carries a live tag. Every address is an ``Addr``: its space picks the
scope.
"""
def __init__(self, tag):
self.tag = tag
def put(self, a, i, v):
bits = v.bitcast(fx.Int32) if isinstance(v, fx.Float32) else fx.Int32(v)
bo.buffer_store(
fx.Vector.from_elements([bits, self.tag], fx.Int32),
rsrc(a.value),
i * 2,
cache_modifier=_SCOPE_CM[a.space],
)
def put_bf(self, a, i, vs):
"""Elements i .. i+len(vs) (2 or 4, i aligned) as packed bf16 pairs, one
store."""
words = []
for j in range_constexpr(len(vs) // 2):
words += [bf16_pair(vs[2 * j], vs[2 * j + 1]).bitcast(fx.Int32), self.tag]
bo.buffer_store(
fx.Vector.from_elements(words, fx.Int32),
rsrc(a.value),
i,
cache_modifier=_SCOPE_CM[a.space],
)
def put_words(self, a, i, words):
"""Raw 32-bit words (1 or 2) at pairs i.. as (word, tag). Integer payloads
(fp8 words) must come this way: through an f32 value a NaN bit pattern is
not kept."""
vals = []
for w in words:
vals += [fx.Int32(w), self.tag]
bo.buffer_store(
fx.Vector.from_elements(vals, fx.Int32),
rsrc(a.value),
i * 2,
cache_modifier=_SCOPE_CM[a.space],
)
def poll(self, specs, batch=POLL_MAX):
"""Batched poll of pairs ``specs`` = [(Addr, pair index, npairs in {1, 2})],
all of one space.
All pairs are re-loaded together while any tag is stale, so a batch costs
one round trip after its last producer lands. Returns the value words."""
if const_expr(len(specs) == 0):
return []
if const_expr(len(specs) > batch):
return self.poll(specs[:batch], batch) + self.poll(specs[batch:], batch)
spaces = {a.space for a, _, _ in specs}
assert len(spaces) == 1, f"one poll batch spans {spaces}"
cm = _SCOPE_CM[spaces.pop()]
bases = [a.value for a, _, _ in specs]
tag = self.tag
def load_all():
words = []
for b, (_, i, n) in zip(bases, specs):
w = fx.Vector(
bo.buffer_load(
rsrc(b),
fx.Int32(i) * 2,
vec_width=2 * n,
dtype=T.i32,
cache_modifier=cm,
)
)
words += [w[e] for e in range(2 * n)]
return fx.Vector.from_elements(words, fx.Int32)
nw = sum(2 * n for _, _, n in specs)
def pending(v):
bad = v[1] != tag
for e in range_constexpr(3, nw, 2):
bad = bad | (v[e] != tag)
return bad
v = _spin(load_all, pending)
outs, e = [], 0
for _, _, n in specs:
outs.append([v[e + 2 * q] for q in range(n)])
e += 2 * n
return outs
@traced
def publish(put, a, i, v, who):
"""This CTA's plain stores made readable by every CTA, then (thread ``who``)
the mailbox pair (``a``, i) := v. The stores are drained first: a flag may
not overtake the data it announces, and a workgroup-scope release fence does
not wait for them (LLVM's AMDGPU memory model omits vmcnt outside tgsplit
mode: the workgroup's waves share one CU, the readers here do not)."""
rocdl.s_waitcnt(vmcnt=0)
gpu.barrier()
if who:
put(a, i, v)