Files

219 lines
6.7 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FP4 codepoint + UE8M0 scale constants for the UltraQuant KV cache.
K/V codes are FP4 E2M1. Per-group scales are UE8M0 (one byte, a power of
two): ``s = 2^round(log2(c · absmax))`` with ``c = 0.156``. Decode
consumes Q as FP8 E4M3 so scaled F8F6F4 MFMA can run natively on CDNA4.
AoS slot layout (group_size=32), per (token, head):
bytes [0 .. D/2) : K codes (FP4 nibbles, 2/byte)
bytes [D/2 .. D/2 + Gk) : K scales (UE8M0, 1 byte × Gk groups)
bytes [D/2 + Gk .. D + Gk) : V codes
bytes [D + Gk .. D + 2*Gk) : V scales
D=256 → Gk=8 → 272 B/slot. D=128 → Gk=4 → 136 B/slot.
No per-token norm-fold. No V rotation. K is Hadamard-rotated at store.
"""
from __future__ import annotations
import math
# ── 15 unique FP4 E2M1 levels, sorted ascending ────
FP4_LEVELS_SORTED: tuple[float, ...] = (
-6.0,
-4.0,
-3.0,
-2.0,
-1.5,
-1.0,
-0.5,
0.0,
0.5,
1.0,
1.5,
2.0,
3.0,
4.0,
6.0,
)
assert len(FP4_LEVELS_SORTED) == 15
# Midpoints between consecutive sorted levels (14 boundaries).
MIDPOINTS_SORTED: tuple[float, ...] = tuple(
(FP4_LEVELS_SORTED[i] + FP4_LEVELS_SORTED[i + 1]) / 2.0
for i in range(len(FP4_LEVELS_SORTED) - 1)
)
assert len(MIDPOINTS_SORTED) == 14
# FP4 E2M1 bit-pattern decode (16 entries indexed by 4-bit code).
# Sign bit (MSB) | 2 exp bits | 1 mantissa bit (LSB).
FP4_BITS_TO_VALUE: tuple[float, ...] = (
0.0,
0.5,
1.0,
1.5,
2.0,
3.0,
4.0,
6.0,
-0.0,
-0.5,
-1.0,
-1.5,
-2.0,
-3.0,
-4.0,
-6.0,
)
assert len(FP4_BITS_TO_VALUE) == 16
# Sorted index → FP4 E2M1 bit pattern (uint4).
SORTED_TO_BITS: tuple[int, ...] = (
0b1111,
0b1110,
0b1101,
0b1100,
0b1011,
0b1010,
0b1001,
0b0000,
0b0001,
0b0010,
0b0011,
0b0100,
0b0101,
0b0110,
0b0111,
)
assert len(SORTED_TO_BITS) == 15
for _i, _bits in enumerate(SORTED_TO_BITS):
_v = FP4_BITS_TO_VALUE[_bits]
_v = 0.0 if _v == 0.0 else _v
assert _v == FP4_LEVELS_SORTED[_i]
# ── Format constants ──────────────────────────────────────────────────────
GROUP_SIZE: int = 32
"""Elements per group; one E8M0 scale per group. Fixed at 32 because the
AMD scaled F8F6F4 MFMA instruction also consumes one E8M0 scale per 32
elements along K. Other group sizes would force a fallback to plain MFMA
+ accumulator-side scale multiply (no hardware fast-path)."""
FP4_MAX: float = 6.0
"""Largest representable FP4 magnitude."""
DEFAULT_CONSTANT_C: float = 0.156
"""MSE-optimal scale constant; ``s_raw = c * absmax``, then UE8M0-snapped."""
def get_constant_c() -> float:
"""Return the fixed UltraQuant scale constant."""
return DEFAULT_CONSTANT_C
def get_group_size() -> int:
"""Return the fixed group size required by scaled MFMA."""
return GROUP_SIZE
# ── Scale encoding ────────────────────────────────────────────────────────
# Per-group scale `s = pow2_round(c · absmax)` stored as the E8M0 byte;
# codebook is the raw FP4 grid {0, ±0.5, ..., ±6}; dequant = code · s.
# ── UE8M0 helpers ──────────────────────────────────────────────────────────
# E8M0 = 8 exponent bits, 0 mantissa bits, no sign. The byte value `e`
# represents `2^(e - 127)` for e in [1, 254]. e=0 and e=255 are reserved
# (zero and NaN respectively in the OCP spec). We use e=0 as our zero
# sentinel for sink-zero groups.
UE8M0_BIAS: int = 127
# fp32 minimum positive normal = 2^-126 ≈ 1.175e-38. Anything smaller
# clamps to the zero sentinel.
_UE8M0_MIN_EXP: int = -126
_UE8M0_MAX_EXP: int = 127
def ue8m0_encode(s: float) -> int:
"""Snap a positive fp32 scale `s` to the nearest power of 2 and
encode as a UE8M0 byte. `s <= 0` encodes as 0 (zero sentinel)."""
if s <= 0.0 or not math.isfinite(s):
return 0
exp = int(round(math.log2(s)))
if exp < _UE8M0_MIN_EXP:
return 0
if exp > _UE8M0_MAX_EXP:
exp = _UE8M0_MAX_EXP
return exp + UE8M0_BIAS
def ue8m0_decode(byte: int) -> float:
"""Decode a UE8M0 byte back to fp32. 0 → 0.0 (zero sentinel)."""
if byte == 0:
return 0.0
return 2.0 ** (byte - UE8M0_BIAS)
# ── Per-slot AoS byte layout ──────────────────────────────────────────────
# Per slot, per head (group_size=GS=32, Gk = D/GS):
# bytes [0 .. D/2) : K codes (2 nibbles/byte)
# bytes [D/2 .. D/2 + Gk) : K scales (E8M0, 1 byte each)
# bytes [D/2 + Gk .. D + Gk) : V codes
# bytes [D + Gk .. D + 2*Gk) : V scales (E8M0, 1 byte each)
# D=256 → 272 B/slot. D=128 → 136 B/slot.
def k_codes_bytes(head_dim: int) -> int:
"""Bytes for one head's packed K codes."""
if head_dim % 2 != 0:
raise ValueError(f"head_dim must be even, got {head_dim}")
return head_dim // 2
def n_groups(head_dim: int, group_size: int | None = None) -> int:
gs = group_size if group_size is not None else get_group_size()
if head_dim % gs != 0:
raise ValueError(f"head_dim={head_dim} must be a multiple of group_size={gs}")
return head_dim // gs
def k_scales_bytes(head_dim: int, group_size: int | None = None) -> int:
"""Bytes for one head's K scales (one E8M0 byte per group)."""
return n_groups(head_dim, group_size)
def v_codes_bytes(head_dim: int) -> int:
return k_codes_bytes(head_dim)
def v_scales_bytes(head_dim: int, group_size: int | None = None) -> int:
return k_scales_bytes(head_dim, group_size)
def slot_size(head_dim: int, group_size: int | None = None) -> int:
"""Bytes per (token, head) slot."""
return (
k_codes_bytes(head_dim)
+ k_scales_bytes(head_dim, group_size)
+ v_codes_bytes(head_dim)
+ v_scales_bytes(head_dim, group_size)
)
def k_codes_offset(head_dim: int, group_size: int | None = None) -> int:
return 0
def k_scales_offset(head_dim: int, group_size: int | None = None) -> int:
return k_codes_bytes(head_dim)
def v_codes_offset(head_dim: int, group_size: int | None = None) -> int:
return k_codes_bytes(head_dim) + k_scales_bytes(head_dim, group_size)
def v_scales_offset(head_dim: int, group_size: int | None = None) -> int:
return v_codes_offset(head_dim, group_size) + v_codes_bytes(head_dim)