[ROCm] [Model] Enable TML inkling (#48841)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
TJian
2026-07-27 10:05:46 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent bc3629b1c4
commit 0906123953
33 changed files with 9213 additions and 9 deletions
@@ -0,0 +1,176 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import statistics
import torch
from tabulate import tabulate
from vllm.models.inkling.nvidia.ops import qkvr_prep
from vllm.utils.argparse_utils import FlexibleArgumentParser
def make_inputs(tokens: int, tp_size: int, is_local: bool):
torch.manual_seed(0)
num_q_heads = 64 // tp_size
num_kv_heads = (16 if is_local else 8) // tp_size
head_dim = 128
d_rel = 16
rel_extent = 512 if is_local else 1024
page_size = 16
num_blocks = (tokens + page_size - 1) // page_size
q_width = num_q_heads * head_dim
kv_width = num_kv_heads * head_dim
r_width = num_q_heads * d_rel
device = "cuda"
qkvr = torch.randn(
tokens,
q_width + 2 * kv_width + r_width,
device=device,
dtype=torch.bfloat16,
)
k_weight = torch.randn(kv_width, 4, device=device, dtype=torch.bfloat16)
v_weight = torch.randn_like(k_weight)
q_norm_weight = torch.randn(head_dim, device=device, dtype=torch.bfloat16)
k_norm_weight = torch.randn_like(q_norm_weight)
rel_proj = torch.randn(d_rel, rel_extent, device=device, dtype=torch.bfloat16)
conv_cache = torch.zeros(
num_blocks,
num_kv_heads,
page_size,
2 * head_dim,
device=device,
dtype=torch.bfloat16,
)
key_cache = torch.empty(
num_blocks,
page_size,
num_kv_heads,
head_dim,
device=device,
dtype=torch.bfloat16,
)
value_cache = torch.empty_like(key_cache)
positions = torch.arange(tokens, device=device, dtype=torch.int64)
block_table = torch.arange(num_blocks, device=device, dtype=torch.int32)[None]
seq_idx = torch.zeros(tokens, device=device, dtype=torch.int32)
slots = torch.arange(tokens, device=device, dtype=torch.int64)
query_start = torch.zeros(tokens, device=device, dtype=torch.int32)
log_scaling = None
if not is_local:
effective_n = (positions + 1).to(torch.float32)
log_scaling = 1.0 + 0.1 * torch.log(torch.clamp(effective_n / 128000, min=1.0))
return (
qkvr,
k_weight,
v_weight,
q_norm_weight,
k_norm_weight,
rel_proj,
1e-6,
num_q_heads,
num_kv_heads,
head_dim,
d_rel,
conv_cache,
key_cache,
value_cache,
positions,
block_table,
seq_idx,
slots,
query_start,
slots,
0,
head_dim,
page_size,
log_scaling,
)
def capture(implementation, inputs):
outputs = []
def run():
outputs[:] = implementation.fused_qkvr_prep(*inputs)
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
for _ in range(3):
run()
torch.cuda.current_stream().wait_stream(stream)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run()
torch.accelerator.synchronize()
return graph, outputs
def time_graph(graph: torch.cuda.CUDAGraph, warmup: int, repeats: int) -> float:
for _ in range(warmup):
graph.replay()
torch.accelerator.synchronize()
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
for _ in range(repeats):
graph.replay()
end.record()
end.synchronize()
return start.elapsed_time(end) * 1000 / repeats
def benchmark(inputs, args) -> float:
graph, _ = capture(qkvr_prep, inputs)
return statistics.median(
time_graph(graph, args.warmup, args.repeats) for _ in range(args.trials)
)
@torch.inference_mode()
def main(args):
rows = []
for tp_size in args.tp_sizes:
for tokens in args.tokens:
for is_local in (True, False):
triton_us = benchmark(make_inputs(tokens, tp_size, is_local), args)
rows.append(
[
tp_size,
tokens,
"local" if is_local else "global",
triton_us,
]
)
print("Inkling QKVR prep (CUDA graph, median latency)")
print(
tabulate(
rows,
headers=[
"TP",
"tokens",
"scope",
"Triton (us)",
],
floatfmt=("d", "d", "", ".2f"),
)
)
if __name__ == "__main__":
parser = FlexibleArgumentParser()
parser.add_argument(
"--tokens",
type=int,
nargs="+",
default=[1 << power for power in range(15)],
)
parser.add_argument("--tp-sizes", type=int, nargs="+", default=[4, 8])
parser.add_argument("--warmup", type=int, default=20)
parser.add_argument("--repeats", type=int, default=200)
parser.add_argument("--trials", type=int, default=5)
main(parser.parse_args())
+17
View File
@@ -0,0 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.platforms import current_platform
@pytest.fixture(autouse=True)
def require_rocm_cdna4() -> None:
if not current_platform.is_rocm():
pytest.skip("requires ROCm")
from vllm.platforms.rocm import on_gfx950
if not on_gfx950():
pytest.skip("requires CDNA4")
@@ -0,0 +1,59 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm guards for shared Inkling model-definition contracts."""
from pathlib import Path
import torch
import vllm.models.inkling as inkling
from vllm.lora.utils import get_supported_lora_modules
from vllm.model_executor.models.interfaces import supports_lora
from vllm.models.inkling.amd import model as amd_model
from vllm.models.inkling.amd.mtp import InklingMTP
def test_backend_neutral_definitions_match_nvidia_reference() -> None:
inkling_dir = Path(amd_model.__file__).parents[1]
for filename in ("logits_processor.py", "mtp.py"):
assert (inkling_dir / "amd" / filename).read_bytes() == (
inkling_dir / "nvidia" / filename
).read_bytes()
def test_rocm_exports_its_aligned_mtp_implementation() -> None:
assert inkling.InklingMTP is InklingMTP
def test_rocm_model_exposes_the_upstream_lora_contract() -> None:
model_cls = amd_model._TmlForCausalLMBase
assert supports_lora(model_cls)
assert model_cls.packed_modules_mapping == {
"qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"],
"w13": ["w1", "w3"],
}
assert model_cls.embedding_modules == {"lm_head": "output_embeddings"}
model = torch.nn.Module()
model.embedding_modules = model_cls.embedding_modules
supported = get_supported_lora_modules(model)
assert "embed_tokens" not in supported
assert "lm_head" in supported
def test_lightseek_bundled_adapter_weights_remain_opt_in() -> None:
mapper = amd_model._TmlForCausalLMBase.hf_to_vllm_mapper
weight = torch.empty(1)
name, mapped_weight = next(
iter(
mapper.apply([("language_model.layers.3.attn.wq_du.lora_A.weight", weight)])
)
)
assert name == "model.layers.3.attn.qkvr.lora_A.weight"
assert mapped_weight.shard_id == 0
assert amd_model._is_peft_adapter_weight(name)
lm_head_name = mapper.apply_list(["language_model.lm_head.lora_B.weight"])
assert lm_head_name == ["lm_head.lora_B.weight"]
@@ -0,0 +1,67 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for Quark OCP MXFP4 weights loaded into AITER MoE layouts."""
from types import SimpleNamespace
import torch
from torch import nn
from vllm.models.inkling.amd.moe import InklingMoE
def _fake_moe() -> tuple[InklingMoE, SimpleNamespace]:
moe = InklingMoE.__new__(InklingMoE)
nn.Module.__init__(moe)
moe.n_routed_experts = 4
routed = SimpleNamespace(
moe_config=SimpleNamespace(
moe_parallel_config=SimpleNamespace(tp_rank=1, tp_size=2)
),
# Logical intermediate per rank is 3; AITER pads it to 4.
w13_weight=nn.Parameter(torch.zeros(4, 8, 2, dtype=torch.uint8), False),
w2_weight=nn.Parameter(torch.zeros(4, 5, 3, dtype=torch.uint8), False),
w13_weight_scale=nn.Parameter(torch.ones(4, 8, 2, dtype=torch.uint8), False),
w2_weight_scale=nn.Parameter(torch.ones(4, 5, 3, dtype=torch.uint8), False),
)
moe.experts = SimpleNamespace(routed_experts=routed)
moe._local_expert_slots = ( # type: ignore[method-assign]
lambda: dict(enumerate(range(4)))
)
return moe, routed
def test_loads_logical_tp_shards_and_preserves_aiter_padding():
moe, routed = _fake_moe()
w13 = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4, 12, 2)
w2 = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4, 5, 4)
moe.load_expert_weight("experts.w13_weight", w13)
moe.load_expert_weight("experts.w2_weight", w2)
# TP rank 1 consumes the second six interleaved gate/up rows. AITER keeps
# one padding row in each destination half and one padding w2 column.
torch.testing.assert_close(routed.w13_weight[:, :3], w13[:, 6:12:2])
torch.testing.assert_close(routed.w13_weight[:, 4:7], w13[:, 7:12:2])
assert torch.count_nonzero(routed.w13_weight[:, 3]) == 0
assert torch.count_nonzero(routed.w13_weight[:, 7]) == 0
torch.testing.assert_close(routed.w2_weight[:, :, :2], w2[:, :, 2:4])
assert torch.count_nonzero(routed.w2_weight[:, :, 2]) == 0
def test_unflattens_quark_scales_and_preserves_scale_padding():
moe, routed = _fake_moe()
flat_w13_scale = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4 * 12, 2)
flat_w2_scale = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4 * 5, 4)
moe.load_expert_weight("experts.w13_weight_scale", flat_w13_scale)
moe.load_expert_weight("experts.w2_weight_scale", flat_w2_scale)
w13_scale = flat_w13_scale.view(4, 12, 2)
w2_scale = flat_w2_scale.view(4, 5, 4)
torch.testing.assert_close(routed.w13_weight_scale[:, :3], w13_scale[:, 6:12:2])
torch.testing.assert_close(routed.w13_weight_scale[:, 4:7], w13_scale[:, 7:12:2])
assert torch.all(routed.w13_weight_scale[:, 3] == 1)
assert torch.all(routed.w13_weight_scale[:, 7] == 1)
torch.testing.assert_close(routed.w2_weight_scale[:, :, :2], w2_scale[:, :, 2:4])
assert torch.all(routed.w2_weight_scale[:, :, 2] == 1)
@@ -0,0 +1,425 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for Inkling's ROCm relative-bias paged attention."""
import pytest
import torch
from vllm.models.inkling.amd.ops.fa4_rel_attention import (
bucket_max_seqlen_q,
inkling_fa4_rel_attention,
use_gfx950_gluon_decode,
use_gfx950_gluon_extend,
)
from vllm.models.inkling.amd.ops.rel_attention_decode import (
decode_split_count,
use_split_kv_decode,
)
HEAD_DIM = 128
DTYPE = torch.bfloat16
def _reference(
q: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
rel_logits: torch.Tensor,
block_table: torch.Tensor,
q_lens: list[int],
kv_lens: list[int],
rel_extent: int,
window_left: int | None,
) -> torch.Tensor:
num_heads = q.shape[1]
num_kv_heads = key_cache.shape[2]
gqa_group = num_heads // num_kv_heads
out: list[torch.Tensor] = []
q_start = 0
for req, (q_len, kv_len) in enumerate(zip(q_lens, kv_lens)):
page_size = key_cache.shape[1]
num_pages = (kv_len + page_size - 1) // page_size
page_ids = block_table[req, :num_pages].long()
k = key_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
v = value_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
k = k.float().repeat_interleave(gqa_group, dim=1)
v = v.float().repeat_interleave(gqa_group, dim=1)
q_req = q[q_start : q_start + q_len].float()
rel_req = rel_logits[q_start : q_start + q_len].float()
scores = torch.einsum("qhd,khd->hqk", q_req, k) / HEAD_DIM
q_pos = torch.arange(q_len, device=q.device)[:, None] + kv_len - q_len
k_pos = torch.arange(kv_len, device=q.device)[None, :]
distance = q_pos - k_pos
rel_idx = distance.clamp(0, rel_extent - 1)
bias = rel_req.permute(1, 0, 2).gather(
2, rel_idx[None].expand(num_heads, -1, -1)
)
in_rel_extent = (distance >= 0) & (distance < rel_extent)
scores += torch.where(in_rel_extent[None], bias, 0.0)
masked = distance < 0
if window_left is not None:
masked |= distance > window_left
scores.masked_fill_(masked[None], float("-inf"))
out.append(torch.einsum("hqk,khd->qhd", scores.softmax(-1), v))
q_start += q_len
return torch.cat(out).to(DTYPE)
def test_query_length_bucket():
assert bucket_max_seqlen_q(1) == 1
assert bucket_max_seqlen_q(17) == 32
assert bucket_max_seqlen_q(33) == 64
def test_split_kv_dispatch_thresholds():
assert not use_split_kv_decode(
max_query_len=4,
max_kv_len=65536,
page_size=128,
window_left=-1,
)
assert not use_split_kv_decode(
max_query_len=1,
max_kv_len=4096,
page_size=16,
window_left=-1,
)
assert use_split_kv_decode(
max_query_len=1,
max_kv_len=8192,
page_size=16,
window_left=-1,
)
assert use_split_kv_decode(
max_query_len=1,
max_kv_len=512,
page_size=128,
window_left=511,
)
assert decode_split_count(512, 511) == 4
assert decode_split_count(65536, -1) == 32
def test_gfx950_gluon_dispatch_is_strict(monkeypatch: pytest.MonkeyPatch):
import vllm.models.inkling.amd.ops.fa4_rel_attention as rel_attention
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
assert use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=16, head_dim=128)
assert not use_gfx950_gluon_decode(max_query_len=4, page_size=128, head_dim=128)
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=256)
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: False)
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
assert use_gfx950_gluon_extend(
max_query_len=4,
max_kv_len=8192,
page_size=128,
head_dim=128,
window_left=-1,
)
assert not use_gfx950_gluon_extend(
max_query_len=4,
max_kv_len=8192,
page_size=128,
head_dim=128,
window_left=511,
)
assert not use_gfx950_gluon_extend(
max_query_len=4,
max_kv_len=8192,
page_size=16,
head_dim=128,
window_left=-1,
)
@pytest.mark.parametrize(
("page_size", "window_left", "max_kv_len"),
[
(16, None, None),
(16, 15, None),
(128, None, 8192),
],
)
@torch.inference_mode()
def test_ragged_multi_page_relative_attention(
page_size: int,
window_left: int | None,
max_kv_len: int | None,
):
"""Covers full/chunked prefill, decode, GQA, and the local window."""
torch.manual_seed(19 + int(window_left is not None))
device = "cuda"
q_lens = [17, 1]
kv_lens = [35, 80]
num_heads = 8
num_kv_heads = 2
rel_extent = 16
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
q = torch.randn(sum(q_lens), num_heads, HEAD_DIM, device=device)
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
key_cache = torch.randn(
1 + len(q_lens) * max_pages,
page_size,
num_kv_heads,
HEAD_DIM,
device=device,
)
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
value_cache = torch.randn_like(key_cache)
rel_logits = torch.randn(
sum(q_lens), num_heads, rel_extent, device=device, dtype=DTYPE
)
block_table = torch.stack(
[
torch.arange(1 + req * max_pages, 1 + (req + 1) * max_pages)
for req in range(len(q_lens))
]
).to(device=device, dtype=torch.int32)
cu_seqlens_q = torch.tensor(
[0, *torch.tensor(q_lens).cumsum(0).tolist()],
device=device,
dtype=torch.int32,
)
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
window_size = (-1, -1) if window_left is None else (window_left, 0)
preallocated = torch.empty_like(q)
actual = inkling_fa4_rel_attention(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=bucket_max_seqlen_q(max(q_lens)),
softmax_scale=1.0 / HEAD_DIM,
causal=True,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
max_kv_len=max_kv_len,
out=preallocated,
)
assert actual.data_ptr() == preallocated.data_ptr()
expected = _reference(
q,
key_cache,
value_cache,
rel_logits,
block_table,
q_lens,
kv_lens,
rel_extent,
window_left,
)
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
@pytest.mark.parametrize(
("page_size", "kv_lens", "num_kv_heads", "rel_extent", "window_left"),
[
(16, [257, 513], 1, 64, None),
(128, [257, 512], 2, 512, 511),
],
)
@torch.inference_mode()
def test_split_kv_decode_matches_reference(
page_size: int,
kv_lens: list[int],
num_kv_heads: int,
rel_extent: int,
window_left: int | None,
):
"""Covers page-16 split-KV and page-128 gfx950 Gluon decode."""
torch.manual_seed(41 + page_size)
device = "cuda"
batch_size = len(kv_lens)
q_lens = [1] * batch_size
num_heads = 8
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
q = torch.randn(batch_size, num_heads, HEAD_DIM, device=device)
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
key_cache = torch.randn(
batch_size * max_pages,
page_size,
num_kv_heads,
HEAD_DIM,
device=device,
)
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
value_cache = torch.randn_like(key_cache)
rel_logits = torch.randn(
batch_size,
num_heads,
rel_extent,
device=device,
dtype=DTYPE,
)
block_table = torch.arange(
batch_size * max_pages,
device=device,
dtype=torch.int32,
).view(batch_size, max_pages)
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
cu_seqlens_q = torch.arange(
batch_size + 1,
device=device,
dtype=torch.int32,
)
window_size = (-1, -1) if window_left is None else (window_left, 0)
max_kv_len = 8192 if page_size == 16 else max(kv_lens)
actual = inkling_fa4_rel_attention(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=1,
softmax_scale=1.0 / HEAD_DIM,
causal=True,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
max_kv_len=max_kv_len,
)
expected = _reference(
q,
key_cache,
value_cache,
rel_logits,
block_table,
q_lens,
kv_lens,
rel_extent,
window_left,
)
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
@pytest.mark.parametrize(
(
"q_len",
"kv_len",
"num_kv_heads",
"rel_extent",
"window_left",
"gluon_enabled",
),
[
(1, 513, 2, 512, 511, True),
(1, 513, 2, 512, 511, False),
(4, 8192, 1, 1024, None, True),
],
)
@torch.inference_mode()
def test_gfx950_page128_packed_kv_views_match_reference(
monkeypatch: pytest.MonkeyPatch,
q_len: int,
kv_len: int,
num_kv_heads: int,
rel_extent: int,
window_left: int | None,
gluon_enabled: bool,
):
"""Cover both Gluon and split-KV against vLLM's packed KV allocation."""
monkeypatch.setenv(
"INKLING_GFX950_GLUON",
"1" if gluon_enabled else "0",
)
torch.manual_seed(71 + q_len)
device = "cuda"
page_size = 128
batch_size = 2
num_heads = 8
q_lens = [q_len] * batch_size
kv_lens = [kv_len] * batch_size
pages_per_req = (kv_len + page_size - 1) // page_size
q = torch.randn(
batch_size * q_len,
num_heads,
HEAD_DIM,
device=device,
dtype=DTYPE,
)
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
# FlashAttentionBackend allocates logical [block, head, page, 2 * dim].
# Inkling transposes to [block, page, head, 2 * dim] and splits K/V,
# producing non-contiguous views whose page/head strides include both.
packed_kv = torch.randn(
batch_size * pages_per_req,
num_kv_heads,
page_size,
2 * HEAD_DIM,
device=device,
dtype=DTYPE,
)
key_cache, value_cache = packed_kv.transpose(1, 2).split(HEAD_DIM, dim=-1)
assert not key_cache.is_contiguous()
assert key_cache.stride() == value_cache.stride()
rel_logits = torch.randn(
batch_size * q_len,
num_heads,
rel_extent,
device=device,
dtype=DTYPE,
)
block_table = torch.arange(
batch_size * pages_per_req,
device=device,
dtype=torch.int32,
).view(batch_size, pages_per_req)
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
cu_seqlens_q = torch.arange(
0,
batch_size * q_len + 1,
q_len,
device=device,
dtype=torch.int32,
)
window_size = (-1, -1) if window_left is None else (window_left, 0)
actual = inkling_fa4_rel_attention(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=q_len,
softmax_scale=1.0 / HEAD_DIM,
causal=True,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
max_kv_len=kv_len,
)
expected = _reference(
q,
key_cache,
value_cache,
rel_logits,
block_table,
q_lens,
kv_lens,
rel_extent,
window_left,
)
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm numerical-equivalence tests for the fused MTP depth-layer input kernel.
``embed_dual_rmsnorm_cat`` must match the unfused module sequence numerically:
each rmsnorm computes in fp32 and rounds to bf16 at the same points as the
vendored ``rmsnorm`` kernel (including the bf16 round-trip between the
chained backbone embed_norm and the depth embed_norm), and the fused row
gather matches ``F.embedding``.
"""
from typing import cast
import pytest
import torch
from vllm.models.inkling.amd.ops.norm import (
embed_dual_rmsnorm_cat,
embed_rmsnorm,
rmsnorm,
)
EPS = 1e-6
VOCAB = 4096
BF16_ATOL = torch.finfo(torch.bfloat16).eps
def _assert_bf16_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
# Different Triton reduction schedules can move a small number of outputs
# by one bf16 epsilon while remaining numerically equivalent.
torch.testing.assert_close(actual, expected, rtol=0, atol=BF16_ATOL)
def _ref(hidden, w_h, w_e, emb, w_pre=None):
if w_pre is not None:
emb = rmsnorm(emb, w_pre, EPS)
return torch.cat([rmsnorm(hidden, w_h, EPS), rmsnorm(emb, w_e, EPS)], dim=-1)
@pytest.mark.parametrize("n", [1536, 6144])
@pytest.mark.parametrize("t", [0, 1, 7, 256])
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
def test_embed_dual_rmsnorm_cat(n: int, t: int, ids_dtype: torch.dtype) -> None:
torch.manual_seed(0)
dev = "cuda"
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
w_h = torch.randn(n, device=dev).to(torch.bfloat16)
w_e = (1 + 0.01 * torch.randn(n, device=dev)).to(torch.bfloat16)
w_pre = torch.randn(n, device=dev).to(torch.bfloat16)
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
hidden = (torch.randn(t, n, device=dev) * 2).to(torch.bfloat16)
emb = table[ids.long()]
# Fused gather + chained backbone pre-norm (the decode draft-step path).
out = embed_dual_rmsnorm_cat(
hidden,
w_h,
w_e,
EPS,
input_ids=ids,
embed_table=table,
pre_norm_weight=w_pre,
)
assert out.shape == (t, 2 * n)
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb, w_pre))
# Precomputed embeds, no pre-norm (draft prefill with target-merged MM
# embeddings, already backbone-normed).
out = embed_dual_rmsnorm_cat(hidden, w_h, w_e, EPS, embeds=emb)
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
# Fused gather, no pre-norm (use_embed_norm=False).
out = embed_dual_rmsnorm_cat(
hidden, w_h, w_e, EPS, input_ids=ids, embed_table=table
)
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
@pytest.mark.parametrize("n", [1536, 6144])
@pytest.mark.parametrize("t", [0, 1, 7, 256])
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
def test_embed_rmsnorm(n: int, t: int, ids_dtype: torch.dtype) -> None:
torch.manual_seed(0)
dev = "cuda"
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
w = torch.randn(n, device=dev).to(torch.bfloat16)
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
ref_emb = table[ids.long()]
# Gather + embed_norm (base model / MTP prefill embed path).
out = cast(torch.Tensor, embed_rmsnorm(ids, table, w, EPS))
assert out.shape == (t, n)
_assert_bf16_close(out, rmsnorm(ref_emb, w, EPS) if t else ref_emb)
# Pure gather (use_embed_norm=False / replicated module forward).
out = cast(torch.Tensor, embed_rmsnorm(ids, table, None, EPS))
assert torch.equal(out, ref_emb)
# Chained first-layer attn_norm (the target text-path forward): one launch
# emits both the residual and layer 0's normed attention input.
w_chain = (1 + 0.05 * torch.randn(n, device=dev)).to(torch.bfloat16)
res, attn_in = cast(
tuple[torch.Tensor, torch.Tensor],
embed_rmsnorm(ids, table, w, EPS, chain_weight=w_chain),
)
ref_res = rmsnorm(ref_emb, w, EPS) if t else ref_emb
_assert_bf16_close(res, ref_res)
_assert_bf16_close(attn_in, rmsnorm(ref_res, w_chain, EPS) if t else ref_res)
# Chained without embed_norm (use_embed_norm=False).
res, attn_in = cast(
tuple[torch.Tensor, torch.Tensor],
embed_rmsnorm(ids, table, None, EPS, chain_weight=w_chain),
)
assert torch.equal(res, ref_emb)
_assert_bf16_close(attn_in, rmsnorm(ref_emb, w_chain, EPS) if t else ref_emb)
@@ -0,0 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import nn
from vllm.models.inkling.amd.sconv_swa_attn import InklingConvState
def test_runtime_sconv_block_size_tracks_unified_cache_page():
"""The cache planner may enlarge W=4 blocks to match attention pages."""
owner = InklingConvState.__new__(InklingConvState)
nn.Module.__init__(owner)
owner.block_size = 4
owner.kv_cache = torch.tensor([])
assert owner.cache_block_size == 4
owner.kv_cache = torch.empty(2, 1, 32, 1024)
assert owner.cache_block_size == 32
+30 -9
View File
@@ -2,12 +2,21 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.platforms import current_platform
if TYPE_CHECKING:
from .nvidia.model import (
InklingForCausalLM,
InklingForConditionalGeneration,
)
from .nvidia.mtp import InklingMTP
if current_platform.is_rocm():
from .amd.model import (
InklingForCausalLM,
InklingForConditionalGeneration,
)
from .amd.mtp import InklingMTP as InklingMTP
else:
from .nvidia.model import (
InklingForCausalLM,
InklingForConditionalGeneration,
)
from .nvidia.mtp import InklingMTP as InklingMTP
__all__ = [
"InklingForConditionalGeneration",
@@ -18,11 +27,23 @@ __all__ = [
def __getattr__(name: str):
if name == "InklingMTP":
from .nvidia import mtp
if current_platform.is_rocm():
from .amd import mtp as amd_mtp
return amd_mtp.InklingMTP
from .nvidia import mtp as nvidia_mtp
return nvidia_mtp.InklingMTP
return mtp.InklingMTP
if name in __all__:
from .nvidia import model
if current_platform.is_rocm():
from .amd import model as amd_model
return getattr(amd_model, name)
from .nvidia import model as nvidia_model
return getattr(nvidia_model, name)
return getattr(model, name)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+328
View File
@@ -0,0 +1,328 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from typing import cast
import torch
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.utils.torch_utils import (
canonicalize_singleton_dim_strides,
kv_cache_dtype_str_to_dtype,
)
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.attention.backends.flash_attn import (
FlashAttentionBackend,
FlashAttentionMetadata,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheSpec,
SlidingWindowSpec,
)
from ..configs import InklingModelConfig
from .layernorm import InklingRMSNorm
from .ops.fa4_rel_attention import (
bucket_max_seqlen_q,
inkling_fa4_num_splits,
inkling_fa4_rel_attention,
)
from .ops.fa4_warmup import InklingFA4WarmupConfig, register_fa4_warmup
from .ops.qkvr_prep import fused_qkvr_prep
from .sconv_swa_attn import _K, _V, InklingConvState, InklingSconvMetadata
from .short_conv import InklingShortConv
def compute_log_scaling_tau(
positions: torch.Tensor, n_floor: int, alpha: float
) -> torch.Tensor:
effective_n = (positions + 1).to(torch.float32)
return 1.0 + alpha * torch.log(torch.clamp(effective_n / float(n_floor), min=1.0))
class RelLogitsProj(nn.Module):
"""Project the per-head relative branch ``r`` to per-distance logits."""
def __init__(self, d_rel: int, rel_extent: int) -> None:
super().__init__()
self.d_rel = d_rel
self.rel_extent = rel_extent
self.proj = nn.Parameter(torch.empty(d_rel, rel_extent), requires_grad=False)
def forward(self, r_out: torch.Tensor) -> torch.Tensor:
# r_out: (T, num_heads, d_rel) -> (T, num_heads, rel_extent)
return torch.einsum("thd,de->the", r_out, self.proj)
class InklingAttention(nn.Module, AttentionLayerBase):
def __init__(
self,
config: InklingModelConfig,
*,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rel_extent: int,
local_extent: int,
is_local: bool,
prefix: str,
quant_config: QuantizationConfig | None = None,
conv_owner: InklingConvState,
) -> None:
super().__init__()
self.prefix = prefix
self.is_local = is_local
self.hidden_size = config.hidden_size
self.head_dim = head_dim
self.d_rel = config.d_rel
self.log_scaling_n_floor = config.log_scaling_n_floor
self.log_scaling_alpha = config.log_scaling_alpha
# q/k are per-head RMS-normed (unit norm), so Inkling scales by 1/head_dim.
self.scaling = 1.0 / head_dim
tp_size = get_tensor_model_parallel_world_size()
self.num_total_heads = num_heads
self.num_total_kv_heads = num_kv_heads
assert self.num_total_heads % tp_size == 0
self.num_heads = self.num_total_heads // tp_size
if self.num_total_kv_heads >= tp_size:
assert self.num_total_kv_heads % tp_size == 0
else:
assert tp_size % self.num_total_kv_heads == 0
self.num_kv_heads = max(1, self.num_total_kv_heads // tp_size)
# When tp_size > num_kv_heads the K/V projections are padded up to
# tp_size heads so each rank gets at least one (GQA replication).
kv_total_for_sizing = max(self.num_total_kv_heads, tp_size)
self.qkvr = MergedColumnParallelLinear(
input_size=config.hidden_size,
output_sizes=[
head_dim * self.num_total_heads,
head_dim * kv_total_for_sizing,
head_dim * kv_total_for_sizing,
self.d_rel * self.num_total_heads,
],
bias=config.q_bias,
quant_config=quant_config,
prefix=f"{prefix}.qkvr",
)
self.wo_ud = RowParallelLinear(
input_size=head_dim * self.num_total_heads,
output_size=config.hidden_size,
bias=config.o_bias,
quant_config=quant_config,
# reduce_results=False: the partial output is all-reduced below
# (one-shot custom AR) so the attention-output sconv can run on the
# full hidden width fused with the residual add + rmsnorm.
reduce_results=False,
prefix=f"{prefix}.wo_ud",
)
self.rel_extent = local_extent if is_local else rel_extent
self.local_extent = local_extent if is_local else None
self.rel_logits_proj = RelLogitsProj(self.d_rel, self.rel_extent)
self.q_norm = InklingRMSNorm(head_dim, eps=config.rms_norm_eps)
self.k_norm = InklingRMSNorm(head_dim, eps=config.rms_norm_eps)
# Short convolution on the K/V streams (per-head-width, TP sharded),
# applied after the qkvr projection and before q/k norm.
kv_conv_dim = self.num_kv_heads * head_dim
self.conv_owner = conv_owner
self.k_sconv = InklingShortConv(
kv_conv_dim, config.sconv_kernel_size, owner=conv_owner, stream_idx=_K
)
self.v_sconv = InklingShortConv(
kv_conv_dim, config.sconv_kernel_size, owner=conv_owner, stream_idx=_V
)
# FA4 left/right window; right=0 keeps it causal. local_extent-1 mirrors
# the source (sliding_window_size - 1).
self.window_size: tuple[int, int] = (
(local_extent - 1, 0) if is_local else (-1, -1)
)
# Static per-layer-type KV length bound for the split heuristic: local
# layers never see more than the sliding window.
vllm_config = get_current_vllm_config()
self._max_kv_len = (
local_extent if is_local else vllm_config.model_config.max_model_len
)
# ---- KV-cache wiring (reuse FlashAttentionBackend for metadata) ----
cache_config = vllm_config.cache_config
self.kv_cache_dtype = (
cache_config.cache_dtype if cache_config is not None else "auto"
)
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
self.kv_cache_dtype, vllm_config.model_config
)
self.register_buffer("k_scale", torch.ones((), dtype=torch.float32))
self.register_buffer("v_scale", torch.ones((), dtype=torch.float32))
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
self.kv_cache = torch.tensor([]) # replaced by bind_kv_cache
register_fa4_warmup(
InklingFA4WarmupConfig(
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
head_dim=self.head_dim,
rel_extent=self.rel_extent,
window_size=self.window_size,
is_local=self.is_local,
max_kv_len=self._max_kv_len,
dtype=vllm_config.model_config.dtype,
kv_dtype=self.kv_cache_torch_dtype,
block_size=vllm_config.cache_config.block_size,
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
max_num_batched_tokens=(
vllm_config.scheduler_config.max_num_batched_tokens
),
)
)
def get_attn_backend(self) -> type[AttentionBackend]:
return FlashAttentionBackend
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
block_size = vllm_config.cache_config.block_size
if self.is_local:
assert self.local_extent is not None
return SlidingWindowSpec(
block_size=block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_dim,
dtype=self.kv_cache_torch_dtype,
sliding_window=self.local_extent,
)
return FullAttentionSpec(
block_size=block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_dim,
dtype=self.kv_cache_torch_dtype,
)
def _split_kv_cache(self) -> tuple[torch.Tensor, torch.Tensor]:
key_cache, value_cache = self.kv_cache.transpose(1, 2).split(
self.head_dim, dim=-1
)
return (
canonicalize_singleton_dim_strides(key_cache),
canonicalize_singleton_dim_strides(value_cache),
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
log_scaling: torch.Tensor | None = None,
) -> torch.Tensor:
num_tokens = hidden_states.shape[0]
qkvr, _ = self.qkvr(hidden_states)
attn_metadata = get_forward_context().attn_metadata
attn_output = torch.empty(
(num_tokens, self.num_heads, self.head_dim),
dtype=qkvr.dtype,
device=qkvr.device,
)
if not isinstance(attn_metadata, dict):
attn_output.zero_()
else:
conv_meta = attn_metadata[self.conv_owner.prefix]
md = attn_metadata[self.prefix]
assert isinstance(conv_meta, InklingSconvMetadata)
fa_md = cast(FlashAttentionMetadata, md)
assert self.kv_cache.numel() > 0
assert self.conv_owner.kv_cache.numel() > 0
# One launch: K/V sconv (conv-cache insert + conv + residual),
# Q/K per-head rmsnorm, and the attention KV-cache write. K/V are
# consumed via the KV cache; only normed q is materialized.
key_cache, value_cache = self._split_kv_cache()
off_k, _ = self.conv_owner.stream_ranges[_K]
off_v, _ = self.conv_owner.stream_ranges[_V]
q, rel_logits = fused_qkvr_prep(
qkvr,
self.k_sconv.weight.squeeze(1),
self.v_sconv.weight.squeeze(1),
self.q_norm.weight,
self.k_norm.weight,
self.rel_logits_proj.proj,
self.q_norm.variance_epsilon,
self.num_heads,
self.num_kv_heads,
self.head_dim,
self.d_rel,
self.conv_owner.kv_cache,
key_cache,
value_cache,
positions,
conv_meta.block_table,
conv_meta.seq_idx,
conv_meta.slot_mapping,
conv_meta.query_start,
fa_md.slot_mapping,
off_k,
off_v,
self.conv_owner.cache_block_size,
log_scaling if not self.is_local else None,
)
q = q.view(num_tokens, self.num_heads, self.head_dim)
self._attention(q, rel_logits, attn_output)
flat = attn_output.view(num_tokens, -1)
output, _ = self.wo_ud(flat)
return output
@eager_break_during_capture
def _attention(
self,
q: torch.Tensor,
rel_logits: torch.Tensor,
output: torch.Tensor,
) -> None:
attn_metadata = get_forward_context().attn_metadata
assert isinstance(attn_metadata, dict)
md = cast(FlashAttentionMetadata, attn_metadata[self.prefix])
nt = md.num_actual_tokens
key_cache, value_cache = self._split_kv_cache()
max_seqlen_q = bucket_max_seqlen_q(md.max_query_len)
num_splits = inkling_fa4_num_splits(
is_local=self.is_local,
batch_size=md.seq_lens.shape[0],
max_query_len=max_seqlen_q,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
max_kv_len=md.max_seq_len,
)
inkling_fa4_rel_attention(
q[:nt],
key_cache,
value_cache,
block_table=md.block_table,
cache_seqlens=md.seq_lens,
cu_seqlens_q=md.query_start_loc,
max_seqlen_q=max_seqlen_q,
softmax_scale=self.scaling,
causal=True,
window_size=self.window_size,
rel_extent=self.rel_extent,
rel_logits=rel_logits[:nt],
num_splits=num_splits,
max_kv_len=md.max_seq_len,
out=output[:nt],
)
+26
View File
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling RMSNorm (no bias, weight-scaled), backed by the vendored Triton kernel."""
from __future__ import annotations
import torch
from torch import nn
from .ops import rmsnorm
class InklingRMSNorm(nn.Module):
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
self.hidden_size = hidden_size
def forward(self, x: torch.Tensor) -> torch.Tensor:
if x.numel() == 0:
return x
original_shape = x.shape
x_2d = x.contiguous().view(-1, self.hidden_size)
y = rmsnorm(x_2d, self.weight, self.variance_epsilon)
return y.view(original_shape)
+129
View File
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling logits processor (muP + LoRA aware).
Inkling divides the final logits by a muP width multiplier
(``logits_mup_width_multiplier``). This applies it two ways, depending on
whether an lm_head LoRA is attached:
* No LoRA: fold ``1/mup`` into the lm_head GEMM alpha (fp32 epilogue) -- no
separate elementwise kernel, no extra rounding, no weight mutation.
* LoRA attached: the LoRA manager wraps this layer in
``LogitsProcessorWithLoRA``, whose ``forward`` calls
``type(base_layer).forward(self=wrapper)`` -- so this ``forward`` runs with
``self`` bound to the wrapper. We detect that via ``base_layer`` and take the
LoRA path: run the wrapper's ``_get_logits`` (base logits + the lm_head LoRA
delta), then divide the full logits by the multiplier so the delta is scaled
too. muP thus composes with the LoRA delta, with the dispatch as the only
model-side branching.
"""
from __future__ import annotations
import torch
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
class InklingLogitsProcessor(LogitsProcessor):
"""``LogitsProcessor`` that applies Inkling's muP logits width multiplier.
Args:
vocab_size: Padded vocabulary size.
org_vocab_size: Unpadded vocabulary size (defaults to ``vocab_size``).
scale: Base logits scale (kept ``1.0`` for the served checkpoint).
logits_as_input: Whether the input is already logits.
soft_cap: Optional logit soft cap (``None`` for the served checkpoint).
logits_mup_width_multiplier: muP width divisor for the final logits;
``None`` or ``0`` disables it.
"""
def __init__(
self,
vocab_size: int,
org_vocab_size: int | None = None,
scale: float = 1.0,
logits_as_input: bool = False,
soft_cap: float | None = None,
logits_mup_width_multiplier: float | None = None,
) -> None:
super().__init__(
vocab_size=vocab_size,
org_vocab_size=org_vocab_size,
scale=scale,
logits_as_input=logits_as_input,
soft_cap=soft_cap,
)
self.logits_mup_width_multiplier = logits_mup_width_multiplier
self._logits_zero: torch.Tensor | None = None
def forward(
self,
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
) -> torch.Tensor | None:
# ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper,
# which calls this forward with ``self`` bound to the wrapper. The
# wrapper is not an ``InklingLogitsProcessor`` instance, so dispatch
# ``_lora_forward`` explicitly through the base_layer's class (it
# provides ``_get_logits``/``logits_as_input``; only ``_lora_forward``
# lives on this class).
if hasattr(self, "base_layer"):
return type(self.base_layer)._lora_forward(
self, lm_head, hidden_states, embedding_bias
)
return self._base_forward(lm_head, hidden_states, embedding_bias)
def _lora_forward(
self,
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
) -> torch.Tensor | None:
# ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits``
# returns the base logits plus the lm_head LoRA delta. Apply the muP
# divisor on the full logits so the LoRA delta is scaled too.
mup_multiplier = self.base_layer.logits_mup_width_multiplier
mup = 1.0 / mup_multiplier if mup_multiplier else None
if self.logits_as_input:
logits = hidden_states
else:
logits = self._get_logits(hidden_states, lm_head, embedding_bias)
# TODO: fuse this multiplication
if logits is not None and mup:
assert self.base_layer.soft_cap is None
assert self.base_layer.scale == 1.0
logits = logits * mup
return logits
def _base_forward(
self,
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
) -> torch.Tensor | None:
mup = self.logits_mup_width_multiplier
if not mup:
return super().forward(lm_head, hidden_states, embedding_bias)
# Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue):
# no separate elementwise kernel, no bf16 rounding of scaled logits, and
# no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no
# soft cap, unit logits scale.
assert self.soft_cap is None
assert self.scale == 1.0
w = lm_head.weight
if self._logits_zero is None:
self._logits_zero = w.new_zeros(1)
logits = torch.addmm(
self._logits_zero,
hidden_states,
w.t(),
beta=0.0,
alpha=1.0 / mup,
)
logits = self._gather_logits(logits)
if logits is not None:
logits = logits[..., : self.org_vocab_size]
return logits
+63
View File
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling dense SwiGLU MLP (also used as the MoE shared expert).
The checkpoint stores the gate/up projection as a single fused, *interleaved*
weight (``[gate0, up0, gate1, up1, ...]``), so we use a plain
``ColumnParallelLinear`` whose contiguous row-sharding keeps each gate/up pair
together, and an interleaved SwiGLU activation.
"""
from __future__ import annotations
import torch
from torch import nn
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
class InklingDenseMLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
*,
use_global_scale: bool = False,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.gate_up_proj = ColumnParallelLinear(
hidden_size,
2 * intermediate_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=False,
prefix=f"{prefix}.down_proj",
)
if use_global_scale:
self.global_scale = nn.Parameter(torch.empty(1), requires_grad=False)
else:
self.global_scale = None
def forward(self, x: torch.Tensor) -> torch.Tensor:
from .ops import silu_and_mul_triton
gate_up, _ = self.gate_up_proj(x)
x = silu_and_mul_triton(gate_up)
x, _ = self.down_proj(x)
if self.global_scale is not None:
x.mul_(self.global_scale)
# TP-partial output: the layer's reduce-scatter fallback consumes it.
return x
+669
View File
@@ -0,0 +1,669 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling model implementation for AMD GPUs."""
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
import regex as re
import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_gather,
tensor_model_parallel_reduce_scatter,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsLoRA,
SupportsMultiModal,
SupportsPP,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
from vllm.models.inkling.common.mm_preprocess import (
InklingDummyInputsBuilder,
InklingMultiModalProcessor,
InklingProcessingInfo,
inkling_audio_enabled,
inkling_vision_enabled,
)
from vllm.models.inkling.common.towers import InklingAudio, InklingVision
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from ..configs import InklingMMConfig, InklingModelConfig
from .attention import InklingAttention, compute_log_scaling_tau
from .layernorm import InklingRMSNorm
from .logits_processor import InklingLogitsProcessor
from .mlp import InklingDenseMLP
from .moe import InklingMoE
from .ops.norm import add_rmsnorm, embed_rmsnorm
from .sconv_swa_attn import _ATTN, _MLP, InklingConvState
from .short_conv import InklingShortConv
def _layer_id(name: str) -> int | None:
m = re.search(r"\.layers\.(\d+)\.", name)
return int(m.group(1)) if m else None
def _sconv_add_norm(
delta: torch.Tensor,
hidden: torch.Tensor,
sconv: InklingShortConv,
norm: InklingRMSNorm | None,
positions: torch.Tensor,
) -> tuple[torch.Tensor | None, torch.Tensor]:
"""``h = hidden + sconv(TP-sum(delta)); y = rmsnorm(h)``.
ROCm uses the portable collective path. The Lamport P2P implementation in
the NVIDIA model relies on CUDA GDC and CUDA-specific symmetric-memory
publication semantics which cannot be linked into a gfx950 HSACO."""
norm_w = norm.weight if norm is not None else None
eps = norm.variance_epsilon if norm is not None else 0.0
# RCCL RS -> shard sconv -> AG -> fused add(+rmsnorm).
shard = tensor_model_parallel_reduce_scatter(delta, dim=-1)
shard = sconv(shard.contiguous(), positions)
full = tensor_model_parallel_all_gather(shard, dim=-1)
if norm is None:
return None, hidden + full
return add_rmsnorm(hidden, full, norm_w, eps)
class InklingDecoderLayer(nn.Module):
def __init__(
self,
config: InklingModelConfig,
layer_id: int,
is_local: bool,
quant_config: QuantizationConfig | None,
prefix: str,
force_dense_mlp: bool = False,
) -> None:
super().__init__()
# Per-layer owner of the conv state as a paged SWA cache. The 4 sconv
# streams (K/V/attn/mlp) are packed head-major into one block and share
# it. Built first so the attention layer can wire its K/V sconv to it.
self.conv_state = InklingConvState(
num_kv_heads=(
config.swa_num_key_value_heads
if is_local
else config.num_key_value_heads
),
head_dim=config.swa_head_dim if is_local else config.head_dim,
hidden_size=config.hidden_size,
kernel_size=config.sconv_kernel_size,
prefix=f"{prefix}.conv_state",
)
self.attn_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.attn = InklingAttention(
config,
num_heads=(
config.swa_num_attention_heads
if is_local
else config.num_attention_heads
),
num_kv_heads=(
config.swa_num_key_value_heads
if is_local
else config.num_key_value_heads
),
head_dim=config.swa_head_dim if is_local else config.head_dim,
rel_extent=config.rel_extent,
local_extent=config.sliding_window_size,
is_local=is_local,
prefix=f"{prefix}.attn",
quant_config=quant_config,
conv_owner=self.conv_state,
)
self.mlp_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
if force_dense_mlp or layer_id < config.dense_mlp_idx:
self.mlp: nn.Module = InklingDenseMLP(
hidden_size=config.hidden_size,
intermediate_size=config.dense_intermediate_size,
use_global_scale=config.use_global_scale,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
else:
# InklingMoE decides per layer (from the checkpoint exclude list)
# whether the routed experts are NVFP4 or bf16; the shared sink
# experts are always bf16.
self.mlp = InklingMoE(
config,
prefix=f"{prefix}.mlp",
quant_config=quant_config,
)
# Short convolution on the attention-output and MLP-output residual
# streams, hidden-sharded: the sublayer outputs are reduce-scattered
# to [T, H/tp], the sconv runs on the shard, and an all-gather
# restores the full residual — all fused with the residual add + next
# rmsnorm via the Lamport P2P kernels for decode-sized batches.
tp_size = get_tensor_model_parallel_world_size()
sconv_dim = config.hidden_size // tp_size
self.attn_sconv = InklingShortConv(
sconv_dim, config.sconv_kernel_size, owner=self.conv_state, stream_idx=_ATTN
)
self.mlp_sconv = InklingShortConv(
sconv_dim, config.sconv_kernel_size, owner=self.conv_state, stream_idx=_MLP
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
pending: tuple[torch.Tensor | None, InklingShortConv] | None = None,
defer_mlp_add: bool = False,
attn_in: torch.Tensor | None = None,
log_scaling: torch.Tensor | None = None,
) -> (
torch.Tensor | tuple[torch.Tensor, tuple[torch.Tensor | None, InklingShortConv]]
):
# The previous sublayer's (pre-reduce, pre-sconv) delta is folded in
# fused with its RS/sconv/AG and this layer's pre-attention rmsnorm.
# A None delta means the partials sit in the NVLS symm buffer.
if pending is None:
if attn_in is None:
# First layer; on the text path attn_norm comes fused with
# the embedding gather (chain_weight in embed_rmsnorm).
attn_in = self.attn_norm(hidden_states)
else:
attn_in, hidden_states = _sconv_add_norm(
pending[0], hidden_states, pending[1], self.attn_norm, positions
)
attn_output = self.attn(positions, attn_in, log_scaling)
mlp_in, hidden_states = _sconv_add_norm(
attn_output, hidden_states, self.attn_sconv, self.mlp_norm, positions
)
mlp_output = self.mlp(mlp_in)
if defer_mlp_add:
# Caller folds mlp_output (pre-reduce, pre-sconv) into the next
# fused sconv+add+rmsnorm.
return hidden_states, (mlp_output, self.mlp_sconv)
# Standalone (MTP) tail: finish the sublayer without a norm.
return _sconv_add_norm(
mlp_output, hidden_states, self.mlp_sconv, None, positions
)[1]
class InklingReplicatedEmbedding(nn.Module):
"""Full-vocab embedding table replicated on every TP rank.
Trades the full table per rank (~2.3 GiB at V=201k / H=6144 bf16, vs a
1/tp shard) for no masked lookup and no per-lookup TP all-reduce — one
all-reduce per MTP draft step plus one per verify pass — and keeps the
full table on-rank for the fused gather+norm kernels (``embed_rmsnorm``,
``embed_dual_rmsnorm_cat``). Bit-exact vs vocab-parallel: the all-reduce
there only ever summed one real row against exact zeros. The LM head
stays vocab-sharded.
"""
def __init__(self, num_embeddings: int, embedding_dim: int) -> None:
super().__init__()
self.weight = nn.Parameter(
torch.empty(num_embeddings, embedding_dim, dtype=torch.get_default_dtype()),
requires_grad=False,
)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
return embed_rmsnorm(input_ids, self.weight, None, 0.0)
class InklingModel(nn.Module):
def __init__(
self,
*,
config: InklingModelConfig,
quant_config: QuantizationConfig | None,
prefix: str,
) -> None:
super().__init__()
self.config = config
self.embed_tokens = InklingReplicatedEmbedding(
config.padded_vocab_size, config.hidden_size
)
self.embed_norm = (
InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
if config.use_embed_norm
else None
)
local_ids = set(config.local_layer_ids)
def get_layer(prefix: str) -> InklingDecoderLayer:
idx = _layer_id(prefix + ".") or int(prefix.split(".")[-1])
return InklingDecoderLayer(
config, idx, idx in local_ids, quant_config, prefix
)
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
)
self.norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.hidden_size
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
# Row gather + embed_norm in one launch.
norm = self.embed_norm
return embed_rmsnorm(
input_ids,
self.embed_tokens.weight,
norm.weight if norm is not None else None,
norm.variance_epsilon if norm is not None else 0.0,
)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
attn_in0: torch.Tensor | None = None
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
# embed_norm was already applied when producing inputs_embeds.
hidden_states = inputs_embeds
else:
# Gather + embed_norm + the first layer's attn_norm, one launch.
norm = self.embed_norm
hidden_states, attn_in0 = embed_rmsnorm(
input_ids,
self.embed_tokens.weight,
norm.weight if norm is not None else None,
self.config.rms_norm_eps,
chain_weight=self.layers[self.start_layer].attn_norm.weight,
)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
log_scaling = None
if self.config.log_scaling_n_floor is not None:
log_scaling = compute_log_scaling_tau(
positions,
self.config.log_scaling_n_floor,
self.config.log_scaling_alpha,
)
pending: tuple[torch.Tensor | None, InklingShortConv] | None = None
for layer in self.layers[self.start_layer : self.end_layer]:
hidden_states, pending = layer(
positions,
hidden_states,
pending=pending,
defer_mlp_add=True,
attn_in=attn_in0,
log_scaling=log_scaling,
)
attn_in0 = None
if not get_pp_group().is_last_rank:
if pending is not None:
hidden_states = _sconv_add_norm(
pending[0], hidden_states, pending[1], None, positions
)[1]
return IntermediateTensors({"hidden_states": hidden_states})
if pending is not None:
# Final RS/sconv/AG + residual add fused with the final rmsnorm.
norm_out = _sconv_add_norm(
pending[0], hidden_states, pending[1], self.norm, positions
)[0]
assert norm_out is not None
return norm_out
return self.norm(hidden_states)
class _TmlForCausalLMBase(nn.Module, SupportsPP, SupportsLoRA):
"""Shared text-backbone causal-LM scaffolding for both entry classes."""
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
".w13_dn": ".gate_up_proj",
".w2_md": ".down_proj",
},
orig_to_new_stacked={
".attn.wq_du.": (".attn.qkvr.", 0),
".attn.wk_dv.": (".attn.qkvr.", 1),
".attn.wv_dv.": (".attn.qkvr.", 2),
".attn.wr_du.": (".attn.qkvr.", 3),
},
orig_to_new_prefix={
"model.llm.layers.": "model.layers.",
"model.llm.embed_norm": "model.embed_norm",
"model.llm.embed": "model.embed_tokens",
"model.llm.norm": "model.norm",
"model.llm.unembed": "lm_head",
"language_model.layers.": "model.layers.",
"language_model.lm_head.": "lm_head.",
},
orig_to_new_suffix={
# NVFP4 scale
".w13_weight.scale": ".w13_weight_scale",
".w13_weight.scale2": ".w13_weight_scale_2",
".w2_weight.scale": ".w2_weight_scale",
".w2_weight.scale2": ".w2_weight_scale_2",
},
)
# Quark uses this mapping when resolving quantization exclusions for the
# four checkpoint attention projections fused into qkvr.
packed_modules_mapping = {
"qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"],
"w13": ["w1", "w3"],
}
embedding_modules = {
"lm_head": "output_embeddings",
}
def _build(
self,
vllm_config: VllmConfig,
text_config: InklingModelConfig,
prefix: str,
) -> None:
quant_config = vllm_config.quant_config
self.config = text_config
# ROCm checkpoints use Quark OCP MXFP4. The global Quark config is
# passed into each routed MoE and its exclusion list keeps the rest of
# Inkling in bf16.
# Read by the MRV2 runner to publish per-request short-conv metadata.
# Short convolution is intrinsic to Inkling, so this is always set.
self.uses_sconv = True
self.model = InklingModel(
config=text_config,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "model"),
)
self.lm_head = ParallelLMHead(
text_config.padded_vocab_size,
text_config.hidden_size,
org_num_embeddings=text_config.padded_vocab_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = InklingLogitsProcessor(
text_config.padded_vocab_size,
org_vocab_size=text_config.vocab_size,
soft_cap=text_config.final_logit_softcapping,
logits_mup_width_multiplier=text_config.logits_mup_width_multiplier,
)
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
self.model.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
return self.model(
input_ids,
positions,
intermediate_tensors,
inputs_embeds,
)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
return _load_inkling_weights(self, weights, self.config)
class InklingForCausalLM(_TmlForCausalLMBase):
"""Text-only entry point (``inkling_model`` checkpoints)."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
self._build(vllm_config, vllm_config.model_config.hf_config, prefix)
@MULTIMODAL_REGISTRY.register_processor(
InklingMultiModalProcessor,
info=InklingProcessingInfo,
dummy_inputs=InklingDummyInputsBuilder,
)
class InklingForConditionalGeneration(_TmlForCausalLMBase, SupportsMultiModal):
"""Top-level (multimodal) entry point.
Builds the vision + audio towers on top of the shared text backbone. Inkling has
NO cross-modal fusion (the vision tower emits one token per patch, the audio
tower one token per frame), so generation reuses the inherited backbone
``forward`` / ``compute_logits`` (the latter already applies muP) and this
class only adds multimodal embedding + merge.
"""
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_prefix={
"model.audio.": "audio.",
"model.visual.": "visual.vision_encoder.",
},
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|content_image|>"
if modality.startswith("audio"):
return "<|content_audio_input|>"
raise ValueError("Only image or audio modality is supported")
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
config: InklingMMConfig = vllm_config.model_config.hf_config
self.visual = (
InklingVision(config.vision_config, prefix=maybe_prefix(prefix, "visual"))
if inkling_vision_enabled(config)
else None
)
self.audio = (
InklingAudio(config.audio_config, prefix=maybe_prefix(prefix, "audio"))
if inkling_audio_enabled(config)
else None
)
self._build(vllm_config, config.text_config, prefix)
# -- multimodal embedding -------------------------------------------
def _process_image_input(
self, pixel_values: Any, num_patches: Any
) -> tuple[torch.Tensor, ...]:
assert self.visual is not None
# pixel_values is a list (per item) of [P_i, 2, P, P, 3] tensors,
# or a single concatenated tensor. Normalize to a flat batch, run the
# tower once, then split back per item.
if isinstance(pixel_values, (list, tuple)):
if not pixel_values:
return ()
sizes = [int(p.shape[0]) for p in pixel_values]
patches = torch.cat(list(pixel_values), dim=0)
else:
patches = pixel_values
sizes = self._sizes_from(num_patches, patches.shape[0])
patches = patches.to(device=self.visual.device, dtype=self.visual.dtype)
embeds = self.visual(patches) # [total_patches, D]
return tuple(embeds.split(sizes))
def _process_audio_input(
self, input_audio_features: Any, num_audio_tokens: Any
) -> tuple[torch.Tensor, ...]:
assert self.audio is not None
if isinstance(input_audio_features, (list, tuple)):
if not input_audio_features:
return ()
sizes = [int(d.shape[0]) for d in input_audio_features]
dmel = torch.cat(list(input_audio_features), dim=0)
else:
dmel = input_audio_features
sizes = self._sizes_from(num_audio_tokens, dmel.shape[0])
dmel = dmel.to(device=self.audio.device)
embeds = self.audio(dmel) # [total_frames, D]
return tuple(embeds.split(sizes))
@staticmethod
def _sizes_from(counts: Any, total: int) -> list[int]:
if counts is None:
return [total]
if isinstance(counts, torch.Tensor):
return [int(c) for c in counts.flatten().tolist()]
if isinstance(counts, (list, tuple)):
flat: list[int] = []
for c in counts:
flat.append(int(c.item()) if isinstance(c, torch.Tensor) else int(c))
return flat
return [int(counts)]
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
# Iterate modalities in a stable order so the returned per-item tensors
# line up with their appearance order; the positional merge in
# embed_input_ids handles actual placement.
pixel_values = kwargs.get("pixel_values")
num_patches = kwargs.get("num_patches")
input_audio_features = kwargs.get("input_audio_features")
num_audio_tokens = kwargs.get("num_audio_tokens")
embeddings: tuple[torch.Tensor, ...] = ()
if pixel_values is not None and self.visual is not None:
embeddings += self._process_image_input(pixel_values, num_patches)
if input_audio_features is not None and self.audio is not None:
embeddings += self._process_audio_input(
input_audio_features, num_audio_tokens
)
return embeddings
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
# Override the base's 1-arg embed_input_ids: the runner calls this 3-arg
# signature for multimodal models. Text embeddings come from the shared
# backbone (which applies embed_norm); MM embeddings are scattered in.
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
# Placeholder ids use unused vocabulary slots and these positions are
# overwritten by MM embeds below.
inputs_embeds = self.model.embed_input_ids(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
assert is_multimodal is not None
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
def get_language_model(self) -> nn.Module:
# This class IS the causal LM (the towers are side branches), so the
# language model is self — callers expect a module exposing ``.model``
# / ``.lm_head`` (e.g. the MTP/eagle loader shares embeddings via
# ``get_language_model().model.embed_tokens``).
return self
# ===========================================================================
# Weight loading
# ===========================================================================
_MOE_EXPERT_WEIGHT_RE = re.compile(
r"^(?P<mlp>.*\.mlp)\.(?P<rest>(?:shared_)?experts\..+)$"
)
def _is_peft_adapter_weight(name: str) -> bool:
return ".lora_A." in name or ".lora_B." in name
def _load_inkling_weights(
module: nn.Module,
weights: Iterable[tuple[str, torch.Tensor]],
config: InklingModelConfig,
) -> set[str]:
moe_modules = {
name: mod for name, mod in module.named_modules() if isinstance(mod, InklingMoE)
}
loaded: set[str] = set()
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
local_ids = set(config.local_layer_ids)
def _iter_loadable_weights() -> Iterable[tuple[str, torch.Tensor]]:
for name, weight in module.hf_to_vllm_mapper.apply(weights):
# LightSeek's MXFP4 conversion bundles a quantized copy of the
# standalone PEFT adapter in the base safetensors index. These are
# not base-model parameters; adapters remain opt-in at serving.
if _is_peft_adapter_weight(name):
continue
shard_id = getattr(weight, "shard_id", None)
# Replicate K/V conv-free GQA heads when tp_size > num_kv_heads.
if (
shard_id in (1, 2)
and name.endswith(".attn.qkvr.weight")
and weight.shape[0] > 0
):
lid = _layer_id(name)
if lid is not None:
is_local = lid in local_ids
n_kv = (
config.swa_num_key_value_heads
if is_local
else config.num_key_value_heads
)
head_dim = config.swa_head_dim if is_local else config.head_dim
if tp_size > n_kv and weight.shape[0] == n_kv * head_dim:
kv_idx = (tp_rank * n_kv) // tp_size
weight = weight.narrow(0, kv_idx * head_dim, head_dim)
weight.shard_id = shard_id
# MoE expert tensors (fused stacked, routed + shared sink): translate
# the checkpoint layout to per-expert FusedMoE loads.
moe_match = _MOE_EXPERT_WEIGHT_RE.match(name)
if moe_match is not None and moe_match.group("mlp") in moe_modules:
moe = moe_modules[moe_match.group("mlp")]
for rel in moe.load_expert_weight(moe_match.group("rest"), weight):
loaded.add(f"{moe_match.group('mlp')}.{rel}")
continue
yield name, weight
loader = AutoWeightsLoader(module, skip_prefixes=["model.mtp."])
loaded |= loader.load_weights(_iter_loadable_weights())
# Post-load MoE fixups (default input scales, zeroed EP-padding experts).
for moe_name, moe in moe_modules.items():
for rel in moe.finalize_load():
loaded.add(f"{moe_name}.{rel}")
return loaded
EntryClass = [InklingForCausalLM, InklingForConditionalGeneration]
+693
View File
@@ -0,0 +1,693 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling mixture-of-experts on vLLM's FusedMoE abstraction.
Overfit to the served checkpoint: sigmoid gate (+ selection bias) top-k over
the routed experts, log-sigmoid renormalization over the k routed + S shared
"sink" logits, scaled by route_scale * global_scale. The routed top-k goes
through vLLM's FusedMoE (which handles TP/EP); the sink experts run in
:class:`InklingSinkExperts` -- replicated across EP ranks (every token
activates every sink) and always bf16 (the checkpoint excludes every
``shared_experts`` from quantization).
MXFP4 routed experts reuse vLLM's Quark OCP MXFP4 fused-MoE method; excluded
(bf16) layers fall back to the unquantized method. The checkpoint's fused
stacked tensors (interleaved gate/up rows, ``.scale`` / ``.scale2`` /
``.input_amax`` aux tensors) are translated to the standard per-expert loads
in :meth:`InklingMoE.load_expert_weight`.
"""
from __future__ import annotations
import math
from typing import TYPE_CHECKING
import torch
from torch import nn
from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.distributed import (
get_dp_group,
get_pcp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.triton_utils import tl, tldevice, triton
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
from vllm.utils.torch_utils import aux_stream
from ..configs import InklingModelConfig
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.routed_experts import (
RoutedExperts,
)
# ---------------------------------------------------------------------------
# Gate / expert selection
# ---------------------------------------------------------------------------
_INKLING_LL_BF16_MAX_TOKENS = 64
_MXFP4_INPUT_SCALE_DENOMINATOR = torch.finfo(torch.float8_e4m3fn).max * 6.0
def _linear_with_fp32_out(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
leading = list(x.shape[:-1])
flat = x.flatten(0, -2)
if (
flat.shape[0] <= _INKLING_LL_BF16_MAX_TOKENS
and flat.dtype == torch.bfloat16
and weight.dtype == torch.bfloat16
and flat.is_cuda
and flat.is_contiguous()
and weight.is_contiguous()
and flat.shape[1] % 8 == 0
and current_platform.has_device_capability(90)
and ll_bf16.is_available()
):
out = ll_bf16.ll_bf16_gemm(flat, weight)
else:
out = torch.mm(flat, weight.T, out_dtype=torch.float32)
return out.view(*leading, weight.shape[0])
@triton.jit(do_not_specialize=["T", "route_scale"])
def _inkling_gate_select_kernel(
logits_ptr, # [T, G] fp32 gate logits (stride_logits_0 may include pad)
bias_ptr, # [R] fp32 selection bias (or 0 ptr if HAS_BIAS=False)
global_scale_ptr, # [1] fp32 (or unused if HAS_GSCALE=False)
ids_ptr, # [T, K + S] int32 out: selected expert ids
weights_ptr, # [T, K + S] fp32 out: renormalized weights
route_scale,
T,
G: tl.constexpr, # total gate experts (routed + shared)
stride_logits_0,
R: tl.constexpr, # routed experts
K: tl.constexpr, # top-k routed
S: tl.constexpr, # shared (sink) experts
HAS_BIAS: tl.constexpr,
HAS_GSCALE: tl.constexpr,
BLOCK_G: tl.constexpr,
):
pid = tl.program_id(0).to(tl.int64)
if pid >= T:
return
offs = tl.arange(0, BLOCK_G)
mask_r = offs < R
logits = tl.load(
logits_ptr + pid * stride_logits_0 + offs,
mask=offs < G,
other=float("-inf"),
).to(tl.float32)
# Selection scores: sigmoid(routed logits) (+ bias), non-routed lanes -inf.
sel = tl.where(mask_r, tl.sigmoid(logits), float("-inf"))
if HAS_BIAS:
bias = tl.load(bias_ptr + offs, mask=mask_r, other=0.0).to(tl.float32)
sel = tl.where(mask_r, sel + bias, float("-inf"))
scale = route_scale
if HAS_GSCALE:
scale = scale * tl.load(global_scale_ptr).to(tl.float32)
# Iterative top-K (K is small); argmax tie-breaks to the lowest index
# (stable ordering).
A: tl.constexpr = K + S
offs_a = tl.arange(0, A)
top_ids = tl.zeros([A], dtype=tl.int32)
active = tl.zeros([A], dtype=tl.float32)
for kk in tl.static_range(K):
idx = tl.argmax(sel, axis=0).to(tl.int32)
raw = tl.max(tl.where(offs == idx, logits, float("-inf")), axis=0)
top_ids = tl.where(offs_a == kk, idx, top_ids)
active = tl.where(offs_a == kk, raw, active)
sel = tl.where(offs == idx, float("-inf"), sel)
if S > 0:
# Shared sink logits sit at the tail of the gate output; their expert
# ids continue after the routed range (R + j).
for jj in tl.static_range(S):
raw = tl.max(tl.where(offs == R + jj, logits, float("-inf")), axis=0)
top_ids = tl.where(offs_a == K + jj, tl.full([], R + jj, tl.int32), top_ids)
active = tl.where(offs_a == K + jj, raw, active)
# Log-sigmoid renormalization over the K + S active logits.
abs_l = tl.abs(active)
min_l = tl.minimum(active, 0.0)
log_probs = min_l - tldevice.log1p(tldevice.exp(-abs_l))
max_lp = tl.max(log_probs, axis=0)
exp_shifted = tldevice.exp(log_probs - max_lp)
sum_exp = tl.sum(exp_shifted, axis=0)
weights = exp_shifted / sum_exp * scale
tl.store(ids_ptr + pid * A + offs_a, top_ids)
tl.store(weights_ptr + pid * A + offs_a, weights)
def inkling_gate_select(
logits: torch.Tensor, # [T, >=G] fp32 (rows may carry GEMM padding)
n_gate_experts: int,
n_routed_experts: int,
topk: int,
n_shared_experts: int,
bias: torch.Tensor | None,
route_scale: float,
global_scale: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Sigmoid + bias + top-k + log-sigmoid renorm; returns (weights, ids)."""
assert logits.dtype == torch.float32
tokens = logits.shape[0]
active = topk + n_shared_experts
topk_ids = torch.empty((tokens, active), dtype=torch.int32, device=logits.device)
topk_weights = torch.empty(
(tokens, active), dtype=torch.float32, device=logits.device
)
if tokens == 0:
return topk_weights, topk_ids
_inkling_gate_select_kernel[(tokens,)](
logits,
bias if bias is not None else logits,
global_scale if global_scale is not None else logits,
topk_ids,
topk_weights,
route_scale,
tokens,
n_gate_experts,
logits.stride(0),
n_routed_experts,
topk,
n_shared_experts,
HAS_BIAS=bias is not None,
HAS_GSCALE=global_scale is not None,
BLOCK_G=triton.next_power_of_2(n_gate_experts),
)
return topk_weights, topk_ids
class InklingGate(nn.Module):
"""Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
global scale (the served checkpoint's only configuration)."""
def __init__(
self,
d_model: int,
n_routed_experts: int,
n_shared_experts: int,
experts_per_token: int,
route_scale: float,
*,
use_global_scale: bool = False,
use_gate_bias: bool = False,
) -> None:
super().__init__()
self.n_routed_experts = n_routed_experts
self.n_shared_experts = n_shared_experts
self.n_total_experts = n_routed_experts + n_shared_experts
self.topk = experts_per_token
self.route_scale = route_scale
padded_experts = self.n_total_experts + (-self.n_total_experts) % 8
self.weight = Parameter(
torch.empty(padded_experts, d_model), requires_grad=False
)
set_weight_attrs(self.weight, {"weight_loader": self._load_weight})
self.global_scale: Parameter | None
if use_global_scale:
self.global_scale = Parameter(
torch.empty(1, dtype=torch.float32), requires_grad=False
)
else:
self.global_scale = None
self.bias: Parameter | None
if use_gate_bias:
self.bias = Parameter(
torch.empty(n_routed_experts, dtype=torch.float32),
requires_grad=False,
)
else:
self.bias = None
@staticmethod
def _load_weight(param: Parameter, loaded_weight: torch.Tensor) -> None:
param.data.zero_()
param.data[: loaded_weight.shape[0]].copy_(loaded_weight)
def compute_logits(self, x: torch.Tensor) -> torch.Tensor:
"""fp32 gate logits [T, n_total_experts + pad] (pad columns are junk)."""
return _linear_with_fp32_out(x, self.weight)
def select_experts(
self, gating_output: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""Full selection: (weights, ids) of [T, K + S]. The first K entries
are the routed top-k; the S trailing entries are the sink gammas."""
return inkling_gate_select(
gating_output,
self.n_total_experts,
self.n_routed_experts,
self.topk,
self.n_shared_experts,
self.bias,
self.route_scale,
self.global_scale,
)
# ---------------------------------------------------------------------------
# MoE layer
# ---------------------------------------------------------------------------
def _inkling_moe_ep_size() -> int:
"""EP size the FusedMoE layer will run with (mirrors
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
expert parallelism is enabled)."""
parallel_config = get_current_vllm_config().parallel_config
if not parallel_config.enable_expert_parallel:
return 1
world = (
get_tensor_model_parallel_world_size()
* get_dp_group().world_size
* get_pcp_group().world_size
)
return world if world > 1 else 1
class InklingSinkExperts(nn.Module):
"""Shared "sink" experts with per-token gammas, in bf16.
Replicated across EP ranks (every token activates every sink, so
EP-sharding them would hotspot the owning rank) and TP-sharded on the
intermediate dim so the output remains a TP-partial sum like the routed
output. The sinks are always bf16 (the checkpoint excludes every
``shared_experts`` from quantization): the experts concatenate into two
plain dense GEMMs with the fused sink epilogue between them.
"""
def __init__(
self, n_experts: int, d_model: int, d_mlp: int, *, prefix: str = ""
) -> None:
super().__init__()
self.n_experts = n_experts
tp_size = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()
intermediate_pp = d_mlp // tp_size
self.w13_weight = Parameter(
torch.empty(n_experts, 2 * intermediate_pp, d_model),
requires_grad=False,
)
self.w2_weight = Parameter(
torch.empty(d_model, n_experts * intermediate_pp),
requires_grad=False,
)
self._unit: torch.Tensor | None = None
def load_weight(self, key: str, weight: torch.Tensor) -> list[str]:
"""Load one checkpoint sink tensor (stacked over the S experts)."""
if key == "w13_weight":
if weight.shape != self.w13_weight.shape:
shard = self.w13_weight.shape[1]
weight = weight.narrow(1, self.tp_rank * shard, shard)
self.w13_weight.data.copy_(weight)
return [key]
assert key == "w2_weight"
shard = self.w2_weight.shape[1] // self.n_experts
shard_start = 0 if weight.shape[2] == shard else self.tp_rank * shard
for expert_idx, expert_weight in enumerate(weight):
local_weight = expert_weight.narrow(1, shard_start, shard)
start = expert_idx * shard
self.w2_weight.data[:, start : start + shard].copy_(local_weight)
return [key]
def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor:
"""``sum_e gammas[:, e] * MLP_e(x)`` (TP-partial along d_mlp)."""
from .ops import sink_silu_mul_epilogue
# One GEMM over the experts' stacked w13 (a view), fused epilogue,
# then one GEMM whose K-reduction over the K-concatenated w2 performs
# the expert sum.
if self._unit is None or self._unit.device != x.device:
self._unit = torch.ones(
self.n_experts, dtype=torch.float32, device=x.device
)
raw = x @ self.w13_weight.view(-1, x.shape[-1]).T # (T, S*2F)
h = sink_silu_mul_epilogue(
raw, self._unit, gammas, self._unit, self.n_experts, x.dtype
)
return h @ self.w2_weight.T # (T, D)
class InklingSinkExpertsLinear(nn.Module):
"""LoRA-capable implementation of the Inkling sink experts."""
def __init__(
self,
n_experts: int,
d_model: int,
d_mlp: int,
*,
prefix: str = "",
) -> None:
super().__init__()
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
self.n_experts = n_experts
self.d_mlp = d_mlp
total = n_experts * d_mlp
self.w13 = MergedColumnParallelLinear(
input_size=d_model,
output_sizes=[total, total],
bias=False,
prefix=f"{prefix}.w13",
)
self.w2 = RowParallelLinear(
input_size=total,
output_size=d_model,
bias=False,
reduce_results=False,
prefix=f"{prefix}.w2",
)
self._w2_input_pp = self.w2.input_size_per_partition
self._col_expert: torch.Tensor | None = None
def _gamma_expand(self, gammas: torch.Tensor) -> torch.Tensor:
if self._col_expert is None or self._col_expert.device != gammas.device:
local = self._w2_input_pp
start = get_tensor_model_parallel_rank() * local
cols = torch.arange(start, start + local, device=gammas.device)
self._col_expert = (cols // self.d_mlp).long()
return gammas[:, self._col_expert]
def load_weight(self, key: str, weight: torch.Tensor) -> list[str]:
if key == "w13_weight":
d_model = weight.shape[-1]
gate = weight[:, 0::2, :].reshape(-1, d_model).contiguous()
up = weight[:, 1::2, :].reshape(-1, d_model).contiguous()
self.w13.weight_loader(self.w13.weight, gate, 0)
self.w13.weight_loader(self.w13.weight, up, 1)
return ["w13.weight"]
w = weight.permute(1, 0, 2).reshape(weight.shape[1], -1).contiguous()
self.w2.weight_loader(self.w2.weight, w)
return ["w2.weight"]
def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.w13(x)
gate, up = gate_up.chunk(2, dim=-1)
hidden_states = torch.nn.functional.silu(gate) * up
hidden_states = (hidden_states * self._gamma_expand(gammas)).to(x.dtype)
output, _ = self.w2(hidden_states)
return output
class InklingMoE(nn.Module):
def __init__(
self,
config: InklingModelConfig,
*,
prefix: str = "",
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
# Overfit to the served checkpoint: sigmoid gate renormalized after
# top-k, shared sink experts, interleaved gate/up checkpoint rows.
assert config.gate_activation == "sigmoid" and config.norm_after_topk
assert config.n_shared_experts > 0 and config.shared_expert_sink
assert config.inference_moe_w13_interleaved
n_routed = config.n_routed_experts
n_shared = config.n_shared_experts
self.n_routed_experts = n_routed
self.gate = InklingGate(
d_model=config.hidden_size,
n_routed_experts=n_routed,
n_shared_experts=n_shared,
experts_per_token=config.num_experts_per_tok,
route_scale=config.route_scale,
use_global_scale=config.use_global_scale,
use_gate_bias=config.use_gate_bias,
)
# TRTLLM MoE kernels assume equal, contiguous per-rank expert slabs
# (local_expert_offset = ep_rank * local_num_experts), so pad the
# expert count to a multiple of the EP size. A no-op for the usual
# power-of-two EP sizes (n_routed is a power of two).
num_experts = n_routed + (-n_routed) % _inkling_moe_ep_size()
# The released MXFP4 checkpoint keeps the first routed-expert layer
# in bf16 and lists its two expert weights explicitly in Quark's
# exclusion list. FusedMoE asks for a quant method at the module
# prefix, so an exact weight exclusion would otherwise be missed and
# the bf16 tensors would be loaded into MXFP4 parameters.
routed_quant_config = quant_config
if quant_config is not None:
excluded = set(getattr(quant_config, "quant_config", {}).get("exclude", ()))
expert_prefix = f"{prefix}.experts"
routed_weights = {
f"{expert_prefix}.w13_weight",
f"{expert_prefix}.w2_weight",
}
if routed_weights <= excluded:
routed_quant_config = None
self.experts = FusedMoE(
num_experts=num_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
renormalize=False,
quant_config=routed_quant_config,
prefix=f"{prefix}.experts",
custom_routing_function=self._select_routed,
router_logits_dtype=torch.float32,
activation="silu",
)
# The decoder layer reduce-scatters the MoE delta into the sconv
# stream itself (RS -> shard sconv -> AG); the runner must return the
# per-rank partial sum instead of all-reducing.
self.experts.moe_config.skip_final_all_reduce = True
self._routed_sel: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None
sink_experts_cls = (
InklingSinkExpertsLinear
if get_current_vllm_config().lora_config is not None
else InklingSinkExperts
)
self.sink_experts = sink_experts_cls(
n_experts=n_shared,
d_model=config.hidden_size,
d_mlp=config.intermediate_size,
prefix=f"{prefix}.shared_experts",
)
# Sink chain overlaps the routed MoE call on the aux stream for
# decode-sized batches (same pattern as the runner's SharedExperts
# multi-stream overlap). The routed GEMM runs on the default stream and
# the sink chain on the aux stream, joined via these two events by
# ``maybe_execute_in_parallel``.
self._sink_stream: torch.cuda.Stream | None = aux_stream()
self._sink_events = (torch.cuda.Event(), torch.cuda.Event())
def _select_routed(
self,
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""FusedMoE ``custom_routing_function``: the routed top-k slice of the
full (routed + sink) selection.
forward() stashes its selection (keyed by logits identity) so the
gate select runs once per layer; the fallback covers paths where the
runner re-derives the logits (e.g. naive DP dispatch).
"""
del hidden_states, renormalize
assert topk == self.gate.topk
cached = self._routed_sel
self._routed_sel = None
if cached is not None and cached[0] is gating_output:
return cached[1], cached[2]
weights, ids = self.gate.select_experts(gating_output)
return weights[:, :topk].contiguous(), ids[:, :topk].contiguous()
def forward(self, x: torch.Tensor) -> torch.Tensor | None:
router_logits = self.gate.compute_logits(x)
num_tokens = x.shape[0]
# One gate select per layer: the routed slice is stashed for the
# routing function inside the FusedMoE op; the sink gammas are the
# trailing columns.
k = self.gate.topk
weights, ids = self.gate.select_experts(router_logits)
self._routed_sel = (
router_logits,
weights[:, :k].contiguous(),
ids[:, :k].contiguous(),
)
gammas = weights[:, k:]
out, sink_out = maybe_execute_in_parallel(
lambda: self.experts(hidden_states=x, router_logits=router_logits),
lambda: self.sink_experts(x, gammas),
self._sink_events[0],
self._sink_events[1],
self._sink_stream
if num_tokens <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
else None,
)
self._routed_sel = None
return out.add_(sink_out)
# -- weight loading ----------------------------------------------------
def _local_expert_slots(self) -> dict[int, int]:
"""Global expert id -> local slot for this rank's expert partition."""
manager = self.experts.routed_experts.expert_map_manager
if manager.expert_map is None:
return {g: g for g in range(manager.global_num_experts)}
emap = manager.expert_map.tolist()
return {g: slot for g, slot in enumerate(emap) if slot >= 0}
def load_expert_weight(self, name: str, weight: torch.Tensor) -> list[str]:
"""Load one checkpoint expert tensor.
``name`` is relative to the mlp module: ``experts.<t>`` (routed
stack) or ``shared_experts.shared_<t>`` (sink experts). Returns the
loaded param names (relative to this module).
"""
if name.startswith("shared_experts."):
key = name.split(".", 1)[1].replace("shared_", "", 1)
return [
f"sink_experts.{p}" for p in self.sink_experts.load_weight(key, weight)
]
experts: RoutedExperts = self.experts.routed_experts
key = name.split(".", 1)[1]
# original_shape is unused by the vLLM serving layout.
if key.endswith(".original_shape"):
return []
if key.endswith(".input_amax"):
projection = "w13" if key.startswith("w13") else "w2"
amax = float(weight.max())
assert math.isfinite(amax) and amax > 0, (
f"bad {projection} input_amax: {amax}"
)
input_scale = getattr(experts, f"{projection}_input_scale")
input_scale.data.fill_(amax / _MXFP4_INPUT_SCALE_DENOMINATOR)
return [f"experts.routed_experts.{projection}_input_scale"]
# Quark's OCP MXFP4 converter stores block scales as a two-dimensional
# tensor with the expert and projection-row dimensions flattened:
# w13: [E * 2I, H / 32], w2: [E * H, I / 32].
# Restore the expert dimension before applying the EP/TP slicing used
# for both packed weights and scales.
if key.endswith("_weight_scale") and weight.ndim == 2:
if weight.shape[0] % self.n_routed_experts != 0:
raise ValueError(
f"cannot unflatten {name} with shape {tuple(weight.shape)} "
f"over {self.n_routed_experts} experts"
)
weight = weight.view(
self.n_routed_experts,
weight.shape[0] // self.n_routed_experts,
weight.shape[1],
)
param = getattr(experts, key)
slots = self._local_expert_slots()
gids = sorted(slots)
lids = [slots[g] for g in gids]
tp_rank = experts.moe_config.moe_parallel_config.tp_rank
tp_size = experts.moe_config.moe_parallel_config.tp_size
if key.endswith("_scale_2"):
# Per-expert scalars, vectorized over the local experts. The
# fused w13 param carries one slot per gate/up half.
vals = weight[gids].float().to(param.device)
param.data[lids] = vals[:, None] if param.data.ndim == 2 else vals
elif key.startswith("w13"):
# Checkpoint w13 rows are interleaved [g0, u0, g1, u1, ...]; the
# fused param layout is [w1(gate); w3(up)]. The TP-local rows form
# one contiguous slab of the interleaved tensor, so upload just
# that slab (a single bounded synchronous H2D; pre-uploading whole
# untrimmed tensors pins the mmap pages of the entire checkpoint
# and OOMs the host) and de-interleave on device.
dst_half = param.shape[1] // 2
if weight.shape[1] % (2 * tp_size) != 0:
raise ValueError(
f"cannot TP-shard {name} with shape {tuple(weight.shape)} "
f"over {tp_size} ranks"
)
logical_half = weight.shape[1] // (2 * tp_size)
if logical_half > dst_half:
raise ValueError(
f"checkpoint shard for {name} has {logical_half} rows per "
f"gate/up half, but destination only has {dst_half}"
)
for gid, lid in slots.items():
slab = weight[gid].narrow(
0,
tp_rank * 2 * logical_half,
2 * logical_half,
)
slab = slab.to(param.device)
param.data[lid, :logical_half].copy_(slab[0::2])
param.data[
lid,
dst_half : dst_half + logical_half,
].copy_(slab[1::2])
else:
# w2: shard the packed intermediate (last) dim. AITER rounds the
# destination intermediate width to 256, so derive the logical TP
# slice from the checkpoint and leave the destination tail at its
# initialized padding value (zero for weights, one for scales).
if weight.shape[2] % tp_size != 0:
raise ValueError(
f"cannot TP-shard {name} with shape {tuple(weight.shape)} "
f"over {tp_size} ranks"
)
shard = weight.shape[2] // tp_size
if shard > param.shape[2]:
raise ValueError(
f"checkpoint shard for {name} has width {shard}, but "
f"destination only has {param.shape[2]}"
)
for gid, lid in slots.items():
param.data[lid, :, :shard].copy_(
weight[gid].narrow(1, tp_rank * shard, shard)
)
return [f"experts.routed_experts.{key}"]
def finalize_load(self) -> list[str]:
"""Post-load fixups for zeroed padding experts."""
experts = self.experts.routed_experts
out: list[str] = []
# Zero the EP-alignment padding experts (if any) so their
# (never-routed) slots hold defined values.
slots = self._local_expert_slots()
for gid in range(self.n_routed_experts, experts.global_num_experts):
lid = slots.get(gid)
if lid is None:
continue
for pname in (
"w13_weight",
"w2_weight",
"w13_weight_scale",
"w2_weight_scale",
"w13_weight_scale_2",
"w2_weight_scale_2",
):
p = getattr(experts, pname, None)
if p is not None:
p.data[lid].zero_()
return out
+410
View File
@@ -0,0 +1,410 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling MTP (Multi-Token Prediction) draft model (NVIDIA).
Implements the first MTP depth from the reference ``mtp_model.py`` shipped with
the checkpoint. It owns ``hidden_norm`` / ``embed_norm`` RMSNorms, a ``2H -> H``
input projection, and a full Inkling transformer block with a dense bf16 MLP.
The draft shares the target's token embedding table and LM head
(``load_eagle_model`` wires those references) and applies the backbone
``embed_norm`` on top: the depth layers were trained on the same normed
embeddings the backbone consumes (their own ``embed_norm`` weights are
near-identity trims, unlike the backbone's whitening ``embed_norm``).
"""
from __future__ import annotations
from collections.abc import Iterable
import regex as re
import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.utils import maybe_prefix
from vllm.sequence import IntermediateTensors
from ..configs import InklingModelConfig
from .layernorm import InklingRMSNorm
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
# Mirrors the backbone's hf_to_vllm_mapper.orig_to_new_stacked; kept as a
# local (pname, wname, shard) list since the MTP loader remaps by hand.
_ATTENTION_PARAMS_MAPPING = [
("qkvr", "wq_du", 0),
("qkvr", "wk_dv", 1),
("qkvr", "wv_dv", 2),
("qkvr", "wr_du", 3),
]
def _mtp_depth_from_name(name: str) -> int | None:
m = re.search(r"\.mtp\.layers\.(\d+)\.", name)
return int(m.group(1)) if m else None
class InklingMTPDepthLayer(nn.Module):
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
super().__init__()
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.input_proj = ReplicatedLinear(
config.hidden_size * 2,
config.hidden_size,
bias=False,
return_bias=False,
prefix=f"{prefix}.input_proj",
)
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
# full attention (the swa_* head config and sliding_window window) to
# match this depth's checkpoint transformer_block weights.
self.transformer_block = InklingDecoderLayer(
config,
layer_id=0,
is_local=is_local,
quant_config=None,
prefix=f"{prefix}.transformer_block",
force_dense_mlp=True,
)
def forward(self, combined: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
# ``combined`` is the fused-normed [rmsnorm(hidden) | embed_norm(emb)]
# input, built by InklingMultiTokenPredictor.fused_input_cat in one launch.
hidden = self.input_proj(combined)
# The short conv self-fetches its paged SWA-cache metadata from the
# forward context (via its conv_owner prefix); no conv_meta to thread.
return self.transformer_block(positions, hidden)
class InklingMultiTokenPredictor(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config: InklingModelConfig = (
vllm_config.speculative_config.draft_model_config.hf_config
)
self.config = config
if vllm_config.speculative_config.num_speculative_tokens != 1:
raise ValueError(
"Inkling MTP currently supports exactly one speculative token"
)
self.chain_hidden_post_norm = config.chain_hidden_post_norm
local_ids = set(config.local_layer_ids)
self.layers = nn.ModuleDict(
{"0": InklingMTPDepthLayer(config, f"{prefix}.layers.0", 0 in local_ids)}
)
self.chain_norm = (
InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
if self.chain_hidden_post_norm
else None
)
# The target's raw token embedding (pre embed_norm), attached by
# load_eagle_model. Never materialized here: building our own
# replicated copy would transiently double the 2.3 GiB table.
self.embed_tokens: InklingReplicatedEmbedding = None # type: ignore[assignment]
# The depth layers consume the *backbone-normed* embedding
# (embed_norm(embed(ids))), not the raw one: mtp embed_norm weights
# are near-identity (trained on already-normalized inputs), and
# feeding raw embeddings drops MTP1 acceptance from ~0.85 to ~0.70.
# Weight loaded from the target's embed_norm.weight; gated like the
# target's InklingModel.embed_norm.
self.backbone_embed_norm = (
InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
if config.use_embed_norm
else None
)
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: object | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
"""Draft-prefill embedding: fused gather + backbone embed_norm, then
the target's tower embeddings scattered in unnormed (the backbone
convention — MM embeds are merged after embed_norm)."""
norm = self.backbone_embed_norm
embeds = embed_rmsnorm(
input_ids,
self.embed_tokens.weight,
norm.weight if norm is not None else None,
norm.variance_epsilon if norm is not None else 0.0,
)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0: # type: ignore[arg-type]
return embeds
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
assert is_multimodal is not None
return _merge_multimodal_embeddings(
inputs_embeds=embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
def fused_input_cat(
self,
layer: InklingMTPDepthLayer,
previous_hidden: torch.Tensor,
input_ids: torch.Tensor,
inputs_embeds: torch.Tensor | None,
) -> torch.Tensor:
"""The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one
launch: embedding row gather + the backbone embed_norm + the depth
embed_norm chain on one side, hidden_norm on the other, written
straight into the cat buffer."""
hidden_w = layer.hidden_norm.weight
embed_w = layer.embed_norm.weight
eps = layer.hidden_norm.variance_epsilon
if inputs_embeds is not None:
# Draft prefill with target-merged MM embeddings (already
# backbone-normed via embed_input_ids); only the depth embed_norm
# remains.
return embed_dual_rmsnorm_cat(
previous_hidden, hidden_w, embed_w, eps, embeds=inputs_embeds
)
return embed_dual_rmsnorm_cat(
previous_hidden,
hidden_w,
embed_w,
eps,
input_ids=input_ids,
embed_table=self.embed_tokens.weight,
pre_norm_weight=(
self.backbone_embed_norm.weight
if self.backbone_embed_norm is not None
else None
),
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
# The draft's short conv is a paged SWA-cache layer (its conv_owner is
# auto-enumerated as a draft attention layer); its per-token metadata is
# built by the speculator's build_attn_metadata and read from the
# forward context, so nothing extra is threaded here.
if spec_step_idx != 0:
raise ValueError("Inkling MTP only supports spec_step_idx=0")
layer = self.layers["0"]
combined = self.fused_input_cat(
layer, previous_hidden_states, input_ids, inputs_embeds
)
hidden = layer(combined, positions)
if self.chain_norm is not None:
hidden = self.chain_norm(hidden)
return hidden
class InklingMTP(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config: InklingModelConfig = (
vllm_config.speculative_config.draft_model_config.hf_config
)
self.config = config
self.model = InklingMultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
# The target's (vocab-sharded) LM head, attached by load_eagle_model;
# never materialized here (same reasoning as model.embed_tokens).
self.lm_head: ParallelLMHead = None # type: ignore[assignment]
self.logits_processor = LogitsProcessor(
config.padded_vocab_size,
org_vocab_size=config.vocab_size,
soft_cap=config.final_logit_softcapping,
)
self._logits_zero: torch.Tensor | None = None
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: object | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
return self.model.embed_input_ids(
input_ids, multimodal_embeddings, is_multimodal=is_multimodal
)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
return self.model(
input_ids,
positions,
hidden_states,
inputs_embeds,
spec_step_idx,
)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor | None:
# The MTP shares the base model's LM head, which is trained on
# ``hidden / mup``-scaled inputs, so apply the same mup scaling here
# for a matching logit scale — folded into the lm_head GEMM alpha
# (fp32 epilogue) like the target's compute_logits. (Argmax-invariant
# for greedy draft sampling, but it matters for the gumbel sampling
# distribution at temperature > 0.)
mup = self.config.logits_mup_width_multiplier
if not mup:
return self.logits_processor(self.lm_head, hidden_states)
assert self.logits_processor.soft_cap is None
assert self.logits_processor.scale == 1.0
w = self.lm_head.weight
if self._logits_zero is None:
self._logits_zero = w.new_zeros(1)
logits = torch.addmm(
self._logits_zero,
hidden_states,
w.t(),
beta=0.0,
alpha=1.0 / mup,
)
logits = self.logits_processor._gather_logits(logits)
if logits is not None:
logits = logits[..., : self.logits_processor.org_vocab_size]
return logits
def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
reduction — no full-vocab logits all-gather. The muP divisor is a
positive scalar, so the argmax is invariant and the scaling is
skipped entirely."""
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
return _load_inkling_mtp_weights(self, weights)
def _load_inkling_mtp_weights(
module: InklingMTP,
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
"""Load ``model.mtp.*`` weights into the MTP module.
Checkpoint keys look like ``model.mtp.chain_norm.weight`` and
``model.mtp.layers.{i}.{...}``. The transformer block reuses the backbone
layer's fused-projection layout, so we apply the same qkvr / gate_up / down
remapping as ``_load_inkling_weights``. Token embedding and LM head are shared
(provided by ``load_eagle_model``) and are not present in mtp.safetensors.
"""
# Per-depth attention is full or sliding-window (config.local_layer_ids);
# each depth's qkvr MergedColumnParallelLinear is built with the matching
# (swa_)num_key_value_heads, and its weight_loader handles the TP sharding.
# The sconv SWA cache pins tp_size <= num_key_value_heads, so tp never
# exceeds a layer's kv-head count and no GQA K/V replication is needed here.
params = dict(module.named_parameters())
loaded: set[str] = set()
def _load(name: str, weight: torch.Tensor, shard_id: object = None) -> bool:
param = params.get(name)
if param is None:
return False
loader = getattr(param, "weight_loader", default_weight_loader)
if shard_id is None:
if loader is default_weight_loader or param.shape == weight.shape:
default_weight_loader(param, weight)
else:
loader(param, weight)
else:
loader(param, weight, shard_id) # type: ignore[call-arg]
loaded.add(name)
return True
for name, weight in weights:
depth = _mtp_depth_from_name(name)
# Token embedding and LM head are never materialized on the draft
# (no params to load into); load_eagle_model attaches the target's.
if name in ("model.llm.embed.weight", "model.llm.unembed.weight"):
continue
# The backbone embed_norm, applied to the shared embedding before the
# depth layers (see InklingMultiTokenPredictor.embed_input_ids). The
# per-depth mtp.layers.{i}.embed_norm keys carry ".mtp." and are loaded
# below. Only the shared backbone key routes here.
if name == "model.llm.embed_norm.weight":
_load("model.backbone_embed_norm.weight", weight)
continue
# Only consume the MTP weights; everything else belongs to the target.
if ".mtp." not in name:
continue
# Only the first checkpoint depth is used for MTP=1.
if depth is not None and depth != 0:
continue
# model.mtp.chain_norm.weight -> model.chain_norm.weight
# model.mtp.layers.{i}.X -> model.layers.{i}.X
original_name = name
name = name.replace(".mtp.layers.", ".layers.").replace(
".mtp.chain_norm.", ".chain_norm."
)
if ".chain_norm." in name and module.model.chain_norm is None:
raise ValueError(
"Inkling checkpoint contains chain_norm weights but "
"chain_hidden_post_norm is disabled."
)
# Fused attention qkvr (wq_du/wk_dv/wv_dv/wr_du -> qkvr).
matched = False
for pname, wname, shard in _ATTENTION_PARAMS_MAPPING:
if f".attn.{wname}." in name:
mapped_name = name.replace(f".{wname}.", f".{pname}.")
if not _load(mapped_name, weight, shard):
raise ValueError(f"Unexpected Inkling MTP weight: {original_name}")
matched = True
break
if matched:
continue
# Dense MLP fused gate/up + down.
if ".mlp.w13_dn.weight" in name:
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
elif ".mlp.w2_md.weight" in name:
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
else:
if name.endswith(".bias") and name not in params:
continue
loaded_weight = _load(name, weight)
if not loaded_weight:
raise ValueError(f"Unexpected Inkling MTP weight: {original_name}")
required = {
name
for name in params
if name.startswith("model.layers.") or name.startswith("model.chain_norm.")
}
if (missing := sorted(required - loaded)) and is_mtp_completeness_check_enabled():
raise ValueError(
"Inkling MTP checkpoint is missing required parameters: "
+ ", ".join(missing)
)
return loaded
EntryClass = [InklingMTP]
+44
View File
@@ -0,0 +1,44 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling kernels (NVIDIA).
``rmsnorm`` / ``sconv`` import eagerly. The SwiGLU kernels and the FA4
relative-attention wrapper are exposed lazily to keep this package's
import path lightweight.
"""
from typing import TYPE_CHECKING
from .norm import add_rmsnorm, rmsnorm
from .sconv import fused_sconv
_LAZY_EXPORTS = {
"silu_and_mul_triton": "silu_and_mul",
"sink_silu_mul_epilogue": "silu_and_mul",
"inkling_fa4_rel_attention": "fa4_rel_attention",
"inkling_rel_attention_split_kv_decode": "rel_attention_decode",
}
if TYPE_CHECKING:
from .fa4_rel_attention import inkling_fa4_rel_attention # noqa: F401
from .silu_and_mul import ( # noqa: F401
silu_and_mul_triton,
sink_silu_mul_epilogue,
)
__all__ = [
"add_rmsnorm",
"rmsnorm",
"fused_sconv",
*sorted(_LAZY_EXPORTS),
]
def __getattr__(name: str):
module = _LAZY_EXPORTS.get(name)
if module is not None:
import importlib
mod = importlib.import_module(f".{module}", __name__)
return getattr(mod, name)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,438 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm paged attention with Inkling's query-dependent relative bias.
The NVIDIA implementation uses the score-mod hook in tml-fa4. ROCm Flash
Attention and AITER do not expose an equivalent hook, so this module implements
the same operation directly in Triton. Query heads belonging to one KV head
are processed together and KV pages are gathered through vLLM's block table.
"""
from __future__ import annotations
import os
from typing import cast
import torch
from vllm.models.inkling.amd.ops.rel_attention_decode import (
inkling_rel_attention_split_kv_decode,
use_split_kv_decode,
)
from vllm.platforms.rocm import on_gfx950
from vllm.triton_utils import tl, triton
def bucket_max_seqlen_q(max_seqlen_q: int) -> int:
"""Round the scheduling bound up to a power of two."""
return 1 << max(0, max_seqlen_q - 1).bit_length()
def use_gfx950_gluon_decode(
*, max_query_len: int, page_size: int, head_dim: int
) -> bool:
"""Use the vendored TokenSpeed CDNA4 decode kernel where it is supported."""
return (
os.getenv("INKLING_GFX950_GLUON", "1") == "1"
and on_gfx950()
and max_query_len == 1
and page_size in (64, 128, 256)
and head_dim in (64, 128)
)
def use_gfx950_gluon_extend(
*,
max_query_len: int,
max_kv_len: int,
page_size: int,
head_dim: int,
window_left: int,
) -> bool:
"""Use Gluon only for the long full-attention extend regime it wins."""
return (
os.getenv("INKLING_GFX950_GLUON", "1") == "1"
and on_gfx950()
and max_query_len > 1
and max_kv_len >= 8192
and window_left < 0
and page_size in (64, 128, 256)
and head_dim in (64, 128)
)
def inkling_fa4_num_splits(
*,
is_local: bool,
batch_size: int,
max_query_len: int,
num_heads: int,
num_kv_heads: int,
max_kv_len: int,
) -> int:
"""Keep the NVIDIA-facing split heuristic as API-compatible metadata.
The ROCm Triton implementation performs online softmax in one program and
does not consume the result. Keeping this function unchanged avoids
platform-specific scheduling branches in :mod:`inkling.amd.attention`.
"""
if is_local:
return 1
q_rows = max_query_len * (num_heads // num_kv_heads)
q_tiles = (q_rows + 255) // 256
base_ctas = batch_size * num_kv_heads * q_tiles
target_ctas = (
256 if q_tiles == 1 and batch_size == 1 else (128 if q_tiles == 1 else 64)
)
max_splits = 128
if q_tiles == 1 and batch_size == 1:
if num_kv_heads == 8:
max_splits = 16
elif num_kv_heads == 4 or max_kv_len <= 8192:
max_splits = 32
elif max_kv_len <= 65536:
max_splits = 64
return max(
1,
min(target_ctas // base_ctas, max_splits, (max_kv_len + 127) // 128),
)
@triton.heuristics(
{
"BLOCK_D": lambda args: triton.next_power_of_2(args["head_dim"]),
"BLOCK_H": lambda args: triton.next_power_of_2(args["gqa_group_size"]),
"BLOCK_QH": lambda args: (
args["BLOCK_Q"] * triton.next_power_of_2(args["gqa_group_size"])
),
}
)
@triton.jit(do_not_specialize_on_alignment=["cache_seqlens", "cu_seqlens_q"])
def _inkling_rel_attention_kernel(
q_ptr, # [total_q, Hq, D]
k_ptr, # [blocks, page, Hkv, D]
v_ptr, # [blocks, page, Hkv, D]
rel_ptr, # [total_q, Hq, rel_extent]
out_ptr, # [total_q, Hq, D]
block_table_ptr, # [batch, max_pages]
cache_seqlens,
cu_seqlens_q,
gqa_group_size,
head_dim,
softmax_scale,
stride_q_t,
stride_q_h,
stride_q_d,
stride_k_b,
stride_k_p,
stride_k_h,
stride_k_d,
stride_v_b,
stride_v_p,
stride_v_h,
stride_v_d,
stride_r_t,
stride_r_h,
stride_r_e,
stride_o_t,
stride_o_h,
stride_o_d,
stride_bt_b,
page_size: tl.constexpr,
rel_extent: tl.constexpr,
window_left: tl.constexpr,
BLOCK_Q: tl.constexpr,
BLOCK_K: tl.constexpr,
BLOCK_D: tl.constexpr,
BLOCK_H: tl.constexpr,
BLOCK_QH: tl.constexpr,
):
# A program owns BLOCK_Q query positions and every query head sharing one
# KV head. This makes the QK/PV operations MFMA-friendly on CDNA.
pid_q = tl.program_id(0)
pid_kh = tl.program_id(1)
pid_b = tl.program_id(2)
q_start = tl.load(cu_seqlens_q + pid_b)
q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start
q_block = pid_q * BLOCK_Q
if q_block >= q_len:
return
kv_len = tl.load(cache_seqlens + pid_b)
prefix_len = kv_len - q_len
q_head_start = pid_kh * gqa_group_size
bt_row = block_table_ptr + pid_b * stride_bt_b
q_block_ptr = tl.make_block_ptr(
base=q_ptr + q_start * stride_q_t + q_head_start * stride_q_h,
shape=(q_len, gqa_group_size, head_dim),
strides=(stride_q_t, stride_q_h, stride_q_d),
offsets=(q_block, 0, 0),
block_shape=(BLOCK_Q, BLOCK_H, BLOCK_D),
order=(2, 1, 0),
)
q = tl.load(q_block_ptr, boundary_check=(0, 1, 2), padding_option="zero")
q = tl.reshape(q, (BLOCK_QH, BLOCK_D))
q_rows = q_block + tl.arange(0, BLOCK_Q)
q_abs = prefix_len + q_rows
q_valid = q_rows < q_len
off_d = tl.arange(0, BLOCK_D)
d_valid = off_d < head_dim
m_i = tl.full((BLOCK_QH,), float("-inf"), dtype=tl.float32)
l_i = tl.zeros((BLOCK_QH,), dtype=tl.float32)
acc = tl.zeros((BLOCK_QH, BLOCK_D), dtype=tl.float32)
log2e: tl.constexpr = 1.4426950408889634
for k_start in range(0, kv_len, BLOCK_K):
k_pos = k_start + tl.arange(0, BLOCK_K)
k_valid = k_pos < kv_len
# Masked lanes still need an in-range page-table load.
safe_pos = tl.minimum(k_pos, tl.maximum(kv_len - 1, 0))
page = tl.load(bt_row + safe_pos // page_size).to(tl.int64)
page_off = safe_pos % page_size
k = tl.load(
k_ptr
+ page[None, :] * stride_k_b
+ page_off[None, :] * stride_k_p
+ pid_kh * stride_k_h
+ off_d[:, None] * stride_k_d,
mask=d_valid[:, None] & k_valid[None, :],
other=0.0,
)
scores = tl.dot(q, k) * (softmax_scale * log2e)
dist = q_abs[:, None] - k_pos[None, :]
score_valid = q_valid[:, None] & k_valid[None, :] & (dist >= 0)
if window_left >= 0:
score_valid &= dist <= window_left
# rel_logits is query- and head-dependent. Expand [Q, K] distance
# indices across the GQA heads, then flatten to match QK's row order.
off_h = tl.arange(0, BLOCK_H)
rel_dist = dist[:, None, :]
rel_valid = (
q_valid[:, None, None]
& (off_h[None, :, None] < gqa_group_size)
& (rel_dist >= 0)
& (rel_dist < rel_extent)
)
safe_dist = tl.maximum(0, tl.minimum(rel_dist, rel_extent - 1))
bias = tl.load(
rel_ptr
+ (q_start + q_rows[:, None, None]) * stride_r_t
+ (q_head_start + off_h[None, :, None]) * stride_r_h
+ safe_dist * stride_r_e,
mask=rel_valid,
other=0.0,
)
bias = tl.reshape(bias, (BLOCK_QH, BLOCK_K))
scores += bias.to(tl.float32) * log2e
score_valid = tl.reshape(
score_valid[:, None, :] & (off_h[None, :, None] < gqa_group_size),
(BLOCK_QH, BLOCK_K),
)
scores = tl.where(score_valid, scores, float("-inf"))
m_ij = tl.maximum(m_i, tl.max(scores, axis=1))
has_scores = tl.sum(score_valid.to(tl.int32), axis=1) > 0
# A sliding-window query can have entire early KV tiles masked. Avoid
# the -inf - -inf NaN in online softmax until its first live tile.
alpha = tl.where(has_scores, tl.exp2(m_i - m_ij), 1.0)
p = tl.where(
score_valid,
tl.exp2(scores - m_ij[:, None]),
0.0,
)
l_i = l_i * alpha + tl.sum(p, axis=1)
acc *= alpha[:, None]
v = tl.load(
v_ptr
+ page[:, None] * stride_v_b
+ page_off[:, None] * stride_v_p
+ pid_kh * stride_v_h
+ off_d[None, :] * stride_v_d,
mask=k_valid[:, None] & d_valid[None, :],
other=0.0,
)
acc += tl.dot(p.to(v.dtype), v)
m_i = m_ij
acc /= l_i[:, None]
acc = tl.reshape(acc, (BLOCK_Q, BLOCK_H, BLOCK_D))
out_block_ptr = tl.make_block_ptr(
base=out_ptr + q_start * stride_o_t + q_head_start * stride_o_h,
shape=(q_len, gqa_group_size, head_dim),
strides=(stride_o_t, stride_o_h, stride_o_d),
offsets=(q_block, 0, 0),
block_shape=(BLOCK_Q, BLOCK_H, BLOCK_D),
order=(2, 1, 0),
)
tl.store(
out_block_ptr,
acc.to(out_ptr.dtype.element_ty),
boundary_check=(0, 1, 2),
)
@torch.no_grad()
def inkling_fa4_rel_attention(
q: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
*,
block_table: torch.Tensor,
cache_seqlens: torch.Tensor,
cu_seqlens_q: torch.Tensor,
max_seqlen_q: int,
softmax_scale: float,
causal: bool,
window_size: tuple[int, int],
rel_extent: int,
rel_logits: torch.Tensor,
num_splits: int = 32,
max_kv_len: int | None = None,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Paged varlen attention with Inkling's relative score modification."""
del num_splits
if not causal or window_size[1] not in (0, -1):
raise NotImplementedError("Inkling ROCm attention requires causal masking")
if q.ndim != 3 or key_cache.ndim != 4 or value_cache.ndim != 4:
raise ValueError("expected q [T,H,D] and paged K/V [B,P,Hkv,D]")
if rel_logits.shape != (q.shape[0], q.shape[1], rel_extent):
raise ValueError(
f"relative logits have shape {tuple(rel_logits.shape)}, expected "
f"{(q.shape[0], q.shape[1], rel_extent)}"
)
num_kv_heads = key_cache.shape[2]
if q.shape[1] % num_kv_heads:
raise ValueError("query heads must be divisible by KV heads")
if out is None:
out = torch.empty_like(q)
gqa_group_size = q.shape[1] // num_kv_heads
batch = cu_seqlens_q.shape[0] - 1
if max_kv_len is None:
max_kv_len = key_cache.shape[0] * key_cache.shape[1]
if use_gfx950_gluon_decode(
max_query_len=max_seqlen_q,
page_size=key_cache.shape[1],
head_dim=q.shape[2],
):
# Import only after the architecture guard. The implementation uses
# gfx950-only CDNA4 Gluon layouts and async-copy operations.
from vllm.models.inkling.amd.ops.gluon.rel_mha_decode_gfx950 import (
gluon_rel_mha_decode_gfx950,
)
return gluon_rel_mha_decode_gfx950(
q,
key_cache,
value_cache,
block_table,
cache_seqlens,
max_kv_len,
rel_logits,
cu_seqlens_q,
max_seqlen_q=max_seqlen_q,
window_left=window_size[0],
softmax_scale=softmax_scale,
out=out,
)
if use_gfx950_gluon_extend(
max_query_len=max_seqlen_q,
max_kv_len=max_kv_len,
page_size=key_cache.shape[1],
head_dim=q.shape[2],
window_left=window_size[0],
):
from vllm.models.inkling.amd.ops.gluon.rel_mha_extend_gfx950 import (
gluon_rel_mha_extend_gfx950,
)
# The copied kernel retains TokenSpeed's cu_seqlens_kv parameter for
# API compatibility but does not consume it.
return cast(
torch.Tensor,
gluon_rel_mha_extend_gfx950(
q,
cu_seqlens_q,
cu_seqlens_q,
key_cache,
value_cache,
block_table,
cache_seqlens,
window_left=window_size[0],
max_seqlen_q=max_seqlen_q,
max_seqlen_k=max_kv_len,
rel_logits=rel_logits,
softmax_scale=softmax_scale,
out=out,
),
)
if use_split_kv_decode(
max_query_len=max_seqlen_q,
max_kv_len=max_kv_len,
page_size=key_cache.shape[1],
window_left=window_size[0],
):
return inkling_rel_attention_split_kv_decode(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
softmax_scale=softmax_scale,
window_left=window_size[0],
rel_extent=rel_extent,
rel_logits=rel_logits,
max_kv_len=max_kv_len,
out=out,
)
block_q = 1 if max_seqlen_q == 1 else 4
grid = (triton.cdiv(max_seqlen_q, block_q), num_kv_heads, batch)
_inkling_rel_attention_kernel[grid](
q,
key_cache,
value_cache,
rel_logits,
out,
block_table,
cache_seqlens,
cu_seqlens_q,
gqa_group_size,
q.shape[2],
softmax_scale,
q.stride(0),
q.stride(1),
q.stride(2),
key_cache.stride(0),
key_cache.stride(1),
key_cache.stride(2),
key_cache.stride(3),
value_cache.stride(0),
value_cache.stride(1),
value_cache.stride(2),
value_cache.stride(3),
rel_logits.stride(0),
rel_logits.stride(1),
rel_logits.stride(2),
out.stride(0),
out.stride(1),
out.stride(2),
block_table.stride(0),
page_size=key_cache.shape[1],
rel_extent=rel_extent,
window_left=window_size[0],
BLOCK_Q=block_q,
BLOCK_K=64,
num_warps=4,
num_stages=1,
)
return out
+35
View File
@@ -0,0 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compatibility shim for the ROCm relative-attention implementation.
ROCm uses a Triton kernel which is compiled by vLLM's normal model warmup. The
NVIDIA path registers ahead-of-time CuTeDSL units; importing that provider on
ROCm would pull in CUDA-only tml-fa4 code.
"""
from __future__ import annotations
from dataclasses import dataclass
import torch
@dataclass(frozen=True)
class InklingFA4WarmupConfig:
num_heads: int
num_kv_heads: int
head_dim: int
rel_extent: int
window_size: tuple[int, int]
is_local: bool
max_kv_len: int
dtype: torch.dtype
kv_dtype: torch.dtype
block_size: int
max_num_reqs: int
max_num_batched_tokens: int
def register_fa4_warmup(config: InklingFA4WarmupConfig) -> None:
"""Triton compilation is triggered by the ordinary vLLM warmup forward."""
del config
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GFX950-only Gluon relative-attention kernels for Inkling."""
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,760 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""rel_mha extend Gluon kernel for AMD GFX950.
This handles ragged, multi-token queries against a paged KV cache. The query
axis is tiled into the MFMA ``M`` dimension (prefill-style): ``BLOCK_M`` query
rows of a single q-head share each paged KV tile, so every KV tile is loaded
once and reused across all rows in the tile. The grid is
``(blocks_per_req, batch, n_heads)`` -- all host-known sizes -- and each program
self-locates its request from ``cu_seqlens_q`` / ``cache_seqlens`` in-kernel, so
the launch stays CUDA-graph static with no device->host sync.
Visibility per query row depends on ``is_causal`` and the optional sliding
window:
* ``is_causal=False``: every query token attends the full visible cache, i.e.
``visible_kv = cache_seqlens[batch]``.
* ``is_causal=True``: query tokens are a causal suffix, so the ``i``-th query
token of a request (0-indexed) attends ``prefix + i + 1`` tokens, where
``prefix = cache_seqlens[batch] - query_len[batch]``.
Causal masking only touches the few KV tiles that reach the diagonal; the long
prefix is a mask-free fast path. Sinks and sliding windows (causal or not) are
applied per tile. This is the sole Gluon extend implementation.
"""
import math
import torch
from vllm.models.inkling.amd.ops.gluon.utils import (
_INV_LN2_VALUE,
_LN2,
InputStrides,
PagedKVStrides,
max,
maximum,
)
from vllm.triton_utils import aggregate, gl, gluon
cdna4 = gl.amd.cdna4
async_copy = cdna4.async_copy
# ===-----------------------------------------------------------------------===#
# Query-batched path
#
# This path tiles BLOCK_M query rows (of a single q-head) into the MFMA M
# dimension -- like the prefill kernel -- so each paged KV tile is loaded once
# and reused across all BLOCK_M rows. KV still comes from the paged cache
# (decode-style async page loads). Causal masking only touches the few KV tiles
# that reach the diagonal; the long prefix is a mask-free fast path.
# ===-----------------------------------------------------------------------===#
_EXTEND_SHORT_Q_BLOCK_M = 64
_EXTEND_SHORT_Q_NUM_WARPS = 2
_EXTEND_LONG_Q_BLOCK_M = 128
_EXTEND_LONG_Q_NUM_WARPS = 4
def _select_extend_tile(max_seqlen_q: int, page_size: int) -> tuple[int, int, int]:
"""Return (BLOCK_M, BLOCK_N, NUM_WARPS) for the given max query length.
Queries that fit in a single short tile use it (least padding, most
occupancy); longer ones use the tall tile that covers more rows per
shared-KV pass.
"""
if max_seqlen_q <= _EXTEND_SHORT_Q_BLOCK_M:
return _EXTEND_SHORT_Q_BLOCK_M, page_size, _EXTEND_SHORT_Q_NUM_WARPS
return _EXTEND_LONG_Q_BLOCK_M, page_size, _EXTEND_LONG_Q_NUM_WARPS
@aggregate
class ExtendConfig:
N_HEADS: gl.constexpr
N_KV_HEADS: gl.constexpr
GROUP_SIZE: gl.constexpr
HEAD_DIM: gl.constexpr
SM_SCALE: gl.constexpr
BLOCK_M: gl.constexpr
BLOCK_N: gl.constexpr
NUM_WARPS: gl.constexpr
PAGE_SIZE: gl.constexpr
PAGE_TABLE_STRIDE: gl.constexpr
IS_CAUSAL: gl.constexpr
HAS_LSE: gl.constexpr
WINDOW_LEFT: gl.constexpr
REL_EXTENT: gl.constexpr
REL_BIAS_QK_SCALE: gl.constexpr
IS_FP8: gl.constexpr
q_strides: InputStrides
rel_strides: InputStrides
k_strides: PagedKVStrides
v_strides: PagedKVStrides
qk_layout: gl.constexpr
pv_layout: gl.constexpr
q_layout: gl.constexpr
k_layout: gl.constexpr
p_layout: gl.constexpr
v_layout: gl.constexpr
load_layout: gl.constexpr
store_layout: gl.constexpr
k_smem_layout: gl.constexpr
v_smem_layout: gl.constexpr
@gluon.constexpr_function
def __init__(
self,
N_HEADS,
N_KV_HEADS,
HEAD_DIM,
SM_SCALE,
BLOCK_M,
BLOCK_N,
NUM_WARPS,
PAGE_SIZE,
PAGE_TABLE_STRIDE,
IS_CAUSAL,
HAS_LSE,
WINDOW_LEFT,
REL_EXTENT,
REL_BIAS_QK_SCALE,
IS_FP8,
q_strides,
rel_strides,
k_strides,
v_strides,
):
assert HEAD_DIM in (64, 128)
assert BLOCK_N == PAGE_SIZE
instr_shape = [32, 32, 16]
mfma_layout = gl.amd.AMDMFMALayout(
version=4,
instr_shape=instr_shape,
transposed=True,
warps_per_cta=[NUM_WARPS, 1],
)
qk_layout = mfma_layout
pv_layout = mfma_layout
# Elements loaded per lane depend on the input dtype, matching qk_kw.
# load_threads is how many lanes span HEAD_DIM.
load_vec = 16 if IS_FP8 else 8
load_threads = HEAD_DIM // load_vec
load_layout = gl.BlockedLayout(
[1, load_vec], [64 // load_threads, load_threads], [NUM_WARPS, 1], [1, 0]
)
# Output is always 16-bit, so a 128-bit store has 8 elements.
# store_threads is how many lanes span HEAD_DIM.
store_vec = 8
store_threads = HEAD_DIM // store_vec
store_layout = gl.BlockedLayout(
[1, store_vec], [64 // store_threads, store_threads], [NUM_WARPS, 1], [1, 0]
)
# Padding interval is 64 lanes * load_vec elems.
pad_interval = 64 * load_vec
# Empirically tuned.
pad_k = 16 if IS_FP8 else 8
pad_v = 16 if IS_FP8 else 32
k_smem_layout = gl.PaddedSharedLayout.with_identity_for(
[[pad_interval, pad_k]], [BLOCK_N, HEAD_DIM], [1, 0]
)
v_smem_layout = gl.PaddedSharedLayout.with_identity_for(
[[pad_interval, pad_v]], [BLOCK_N, HEAD_DIM], [1, 0]
)
self.N_HEADS = gl.constexpr(N_HEADS)
self.N_KV_HEADS = gl.constexpr(N_KV_HEADS)
self.GROUP_SIZE = gl.constexpr(N_HEADS // N_KV_HEADS)
self.HEAD_DIM = gl.constexpr(HEAD_DIM)
self.SM_SCALE = gl.constexpr(SM_SCALE)
self.BLOCK_M = gl.constexpr(BLOCK_M)
self.BLOCK_N = gl.constexpr(BLOCK_N)
self.NUM_WARPS = gl.constexpr(NUM_WARPS)
self.PAGE_SIZE = gl.constexpr(PAGE_SIZE)
self.PAGE_TABLE_STRIDE = gl.constexpr(PAGE_TABLE_STRIDE)
self.IS_CAUSAL = gl.constexpr(IS_CAUSAL)
self.HAS_LSE = gl.constexpr(HAS_LSE)
self.WINDOW_LEFT = gl.constexpr(WINDOW_LEFT)
self.REL_EXTENT = gl.constexpr(REL_EXTENT)
self.REL_BIAS_QK_SCALE = gl.constexpr(REL_BIAS_QK_SCALE)
self.IS_FP8 = gl.constexpr(IS_FP8)
self.q_strides = q_strides
self.rel_strides = rel_strides
self.k_strides = k_strides
self.v_strides = v_strides
self.qk_layout = gl.constexpr(qk_layout)
self.pv_layout = gl.constexpr(pv_layout)
# qk_kw is derived from a 128-bit load / dtype bitwidth.
# pv_kw is empirically tuned.
qk_kw = 16 if IS_FP8 else 8
pv_kw = 16 if IS_FP8 else 4
self.q_layout = gl.constexpr(gl.DotOperandLayout(0, qk_layout, k_width=qk_kw))
self.k_layout = gl.constexpr(gl.DotOperandLayout(1, qk_layout, k_width=qk_kw))
self.p_layout = gl.constexpr(gl.DotOperandLayout(0, pv_layout, k_width=pv_kw))
self.v_layout = gl.constexpr(gl.DotOperandLayout(1, pv_layout, k_width=pv_kw))
self.load_layout = gl.constexpr(load_layout)
self.store_layout = gl.constexpr(store_layout)
self.k_smem_layout = gl.constexpr(k_smem_layout)
self.v_smem_layout = gl.constexpr(v_smem_layout)
@aggregate
class ExtendProgram:
cfg: gl.constexpr
q_ptr: gl.tensor
rel_logits_ptr: gl.tensor
k_cache_ptr: gl.tensor
v_cache_ptr: gl.tensor
page_table_ptr: gl.tensor
output_ptr: gl.tensor
lse_ptr: gl.tensor
batch: gl.tensor
q_head: gl.tensor
kv_head: gl.tensor
q_start: gl.tensor
seq_base: gl.tensor
seq_len: gl.tensor
prefix: gl.tensor
cache_len: gl.tensor
@gluon.constexpr_function
def __init__(
self,
cfg,
q_ptr,
rel_logits_ptr,
k_cache_ptr,
v_cache_ptr,
page_table_ptr,
output_ptr,
lse_ptr,
batch,
q_head,
kv_head,
q_start,
seq_base,
seq_len,
prefix,
cache_len,
):
self.cfg = gl.constexpr(cfg)
self.q_ptr = q_ptr
self.rel_logits_ptr = rel_logits_ptr
self.k_cache_ptr = k_cache_ptr
self.v_cache_ptr = v_cache_ptr
self.page_table_ptr = page_table_ptr
self.output_ptr = output_ptr
self.lse_ptr = lse_ptr
self.batch = batch
self.q_head = q_head
self.kv_head = kv_head
self.q_start = q_start
self.seq_base = seq_base
self.seq_len = seq_len
self.prefix = prefix
self.cache_len = cache_len
@gluon.jit
def create(
cfg,
q_ptr,
rel_logits_ptr,
k_cache_ptr,
v_cache_ptr,
page_table_ptr,
output_ptr,
lse_ptr,
cu_seqlens_q_ptr,
cache_seqlens_ptr,
):
# Gluon treats ``cfg`` as the constexpr factory argument, while mypy
# models the first parameter of this aggregate method as ``self``.
block_in_req = gl.program_id(0)
batch = gl.program_id(1)
q_head = gl.program_id(2)
q_start = block_in_req * cfg.BLOCK_M # type: ignore[attr-defined]
seq_base = gl.load(cu_seqlens_q_ptr + batch)
seq_end = gl.load(cu_seqlens_q_ptr + batch + 1)
seq_len = seq_end - seq_base
cache_len = gl.load(cache_seqlens_ptr + batch)
prefix = cache_len - seq_len
kv_head = q_head // cfg.GROUP_SIZE # type: ignore[attr-defined]
return ExtendProgram(
gl.constexpr(cfg),
q_ptr,
rel_logits_ptr,
k_cache_ptr,
v_cache_ptr,
page_table_ptr,
output_ptr,
lse_ptr,
batch,
q_head,
kv_head,
q_start,
seq_base,
seq_len,
prefix,
cache_len,
)
@gluon.jit
def load_q(self):
cfg = self.cfg
offs_m = self.q_start + gl.arange(
0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.q_layout)
)
offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.q_layout))
row = self.seq_base + offs_m
offsets = cfg.q_strides.offsets(row[:, None], self.q_head, offs_d[None, :])
mask = offs_m[:, None] < self.seq_len
return cdna4.buffer_load(self.q_ptr, offsets, mask=mask, other=0.0)
@gluon.jit
def load_page(self, start_n):
cfg = self.cfg
page_index = start_n // cfg.PAGE_SIZE
valid = start_n < self.cache_len
return gl.load(
self.page_table_ptr + self.batch * cfg.PAGE_TABLE_STRIDE + page_index,
mask=valid,
other=0,
)
@gluon.jit
def issue_load_k(self, physical_page, k_smem):
cfg = self.cfg
offs_n = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(1, cfg.load_layout))
offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.load_layout))
offsets = cfg.k_strides.offsets(
physical_page,
offs_n[:, None],
self.kv_head,
offs_d[None, :],
)
async_copy.global_load_to_shared(k_smem, self.k_cache_ptr + offsets)
async_copy.commit_group()
@gluon.jit
def issue_load_v(self, physical_page, v_smem):
cfg = self.cfg
offs_n = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(1, cfg.load_layout))
offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.load_layout))
offsets = cfg.v_strides.offsets(
physical_page,
offs_n[:, None],
self.kv_head,
offs_d[None, :],
)
async_copy.global_load_to_shared(v_smem, self.v_cache_ptr + offsets)
async_copy.commit_group()
@gluon.jit
def shared_load_k(self, k_smem):
cfg = self.cfg
k_buffer = k_smem.permute([1, 0])
return k_buffer.load(cfg.k_layout)
@gluon.jit
def shared_load_v(self, v_smem):
cfg = self.cfg
return v_smem.load(cfg.v_layout)
@gluon.jit
def compute_qk(self, q, k):
cfg = self.cfg
qk = gl.zeros(
[cfg.BLOCK_M, cfg.BLOCK_N], dtype=gl.float32, layout=cfg.qk_layout
)
return cdna4.mfma(q, k, qk)
@gluon.jit
def apply_rel_bias(self, qk, start_n):
cfg = self.cfg
offs_m = self.q_start + gl.arange(
0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.qk_layout)
)
offs_n_abs = start_n + gl.arange(
0, cfg.BLOCK_N, layout=gl.SliceLayout(0, cfg.qk_layout)
)
q_pos = self.prefix + offs_m
rel_dist = q_pos[:, None] - offs_n_abs[None, :]
rel_valid = (rel_dist >= 0) & (rel_dist < cfg.REL_EXTENT)
rel_idx = gl.where(rel_dist >= 0, rel_dist, 0)
rel_idx = gl.where(rel_idx < cfg.REL_EXTENT, rel_idx, cfg.REL_EXTENT - 1)
offsets = cfg.rel_strides.offsets(
self.seq_base + offs_m[:, None], self.q_head, rel_idx
)
mask = (offs_m[:, None] < self.seq_len) & rel_valid
rel_bias = cdna4.buffer_load(
self.rel_logits_ptr, offsets, mask=mask, other=0.0
).to(gl.float32)
return qk + rel_bias * cfg.REL_BIAS_QK_SCALE
@gluon.jit
def compute_pv(self, p, v, acc):
return cdna4.mfma(p, v, acc)
@gluon.jit
def init_attention_state(self):
cfg = self.cfg
m_i = gl.full(
[cfg.BLOCK_M],
value=-float("inf"),
dtype=gl.float32,
layout=gl.SliceLayout(1, cfg.pv_layout),
)
l_i = gl.full(
[cfg.BLOCK_M],
value=0,
dtype=gl.float32,
layout=gl.SliceLayout(1, cfg.pv_layout),
)
acc = gl.zeros(
[cfg.BLOCK_M, cfg.HEAD_DIM], dtype=gl.float32, layout=cfg.pv_layout
)
return m_i, l_i, acc
@gluon.jit
def softmax(self, qk, m_i, l_i, acc):
cfg = self.cfg
# In sliding window case, some rows can see fully masked tiles before
# any valid KV. Guard the online softmax state so `-inf - -inf` does not
# produce NaNs.
HAS_INVALID: gl.constexpr = cfg.WINDOW_LEFT >= 0
row_max = max(qk, 1)
m_new = maximum(m_i, row_max)
m_new_scaled = m_new * cfg.SM_SCALE
if HAS_INVALID:
invalid = m_new == -float("inf")
m_new_scaled = gl.where(invalid, 0.0, m_new_scaled)
qk_shifted = qk * cfg.SM_SCALE - m_new_scaled[:, None]
p = gl.exp2(qk_shifted)
m_diff = m_i * cfg.SM_SCALE - m_new_scaled
if HAS_INVALID:
m_diff = gl.where(invalid, 0.0, m_diff)
alpha = gl.exp2(m_diff)
l_ij = gl.sum(p, axis=1)
l_i = l_i * alpha + l_ij
acc = acc * alpha[:, None]
p = p.to(self.q_ptr.dtype.element_ty)
p = gl.convert_layout(p, cfg.p_layout)
return p, m_new, l_i, acc
@gluon.jit
def store_output(self, output):
cfg = self.cfg
offs_m = self.q_start + gl.arange(
0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.store_layout)
)
offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.store_layout))
offsets = (
((self.seq_base + offs_m[:, None]) * cfg.N_HEADS + self.q_head)
* cfg.HEAD_DIM
+ offs_d[None, :]
).to(gl.int32)
mask = offs_m[:, None] < self.seq_len
output = output.to(self.output_ptr.dtype.element_ty)
cdna4.buffer_store(output, self.output_ptr, offsets, mask=mask)
@gluon.jit
def store_lse(self, l_i, m_i):
cfg = self.cfg
if cfg.HAS_LSE:
offs_m = self.q_start + gl.arange(
0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.pv_layout)
)
offsets = ((self.seq_base + offs_m) * cfg.N_HEADS + self.q_head).to(
gl.int32
)
mask = offs_m < self.seq_len
lse_l_i = gl.where(l_i > 0.0, l_i, 1.0)
# Softmax runs in base-2 (exp2 hardware fast path), so m_i*SM_SCALE +
# log2(l_i) is the LSE in base-2 units. Convert to natural log (the
# public op contract / torch.logsumexp convention) by scaling by ln2.
lse = (m_i * cfg.SM_SCALE + gl.log2(lse_l_i)) * _LN2
cdna4.buffer_store(lse, self.lse_ptr, offsets, mask=mask)
@gluon.jit
def _rel_mha_extend_fp16(
q_ptr,
rel_logits_ptr,
k_cache_ptr,
v_cache_ptr,
page_table_ptr,
output_ptr,
lse_ptr,
cu_seqlens_q_ptr,
cache_seqlens_ptr,
Q_STRIDE_T: gl.constexpr,
Q_STRIDE_H: gl.constexpr,
Q_STRIDE_D: gl.constexpr,
K_STRIDE_B: gl.constexpr,
K_STRIDE_P: gl.constexpr,
K_STRIDE_H: gl.constexpr,
K_STRIDE_D: gl.constexpr,
V_STRIDE_B: gl.constexpr,
V_STRIDE_P: gl.constexpr,
V_STRIDE_H: gl.constexpr,
V_STRIDE_D: gl.constexpr,
SM_SCALE: gl.constexpr,
PAGE_TABLE_STRIDE: gl.constexpr,
PAGE_SIZE: gl.constexpr,
N_HEADS: gl.constexpr,
N_KV_HEADS: gl.constexpr,
HEAD_DIM: gl.constexpr,
BLOCK_M: gl.constexpr,
BLOCK_N: gl.constexpr,
NUM_WARPS: gl.constexpr,
IS_CAUSAL: gl.constexpr,
HAS_LSE: gl.constexpr,
WINDOW_LEFT: gl.constexpr,
REL_STRIDE_T: gl.constexpr,
REL_STRIDE_H: gl.constexpr,
REL_STRIDE_E: gl.constexpr,
REL_EXTENT: gl.constexpr,
REL_BIAS_QK_SCALE: gl.constexpr,
IS_FP8: gl.constexpr,
):
cfg = ExtendConfig(
N_HEADS,
N_KV_HEADS,
HEAD_DIM,
SM_SCALE,
BLOCK_M,
BLOCK_N,
NUM_WARPS,
PAGE_SIZE,
PAGE_TABLE_STRIDE,
IS_CAUSAL,
HAS_LSE,
WINDOW_LEFT,
REL_EXTENT,
REL_BIAS_QK_SCALE,
IS_FP8,
InputStrides(Q_STRIDE_T, Q_STRIDE_H, Q_STRIDE_D),
InputStrides(REL_STRIDE_T, REL_STRIDE_H, REL_STRIDE_E),
PagedKVStrides(K_STRIDE_B, K_STRIDE_P, K_STRIDE_H, K_STRIDE_D),
PagedKVStrides(V_STRIDE_B, V_STRIDE_P, V_STRIDE_H, V_STRIDE_D),
)
program = ExtendProgram.create(
cfg,
q_ptr,
rel_logits_ptr,
k_cache_ptr,
v_cache_ptr,
page_table_ptr,
output_ptr,
lse_ptr,
cu_seqlens_q_ptr,
cache_seqlens_ptr,
)
# Over-provisioned tile past this request's real query rows: nothing to do.
if program.q_start >= program.seq_len:
return
k_smem = gl.allocate_shared_memory(
k_cache_ptr.dtype.element_ty, [cfg.BLOCK_N, cfg.HEAD_DIM], cfg.k_smem_layout
)
v_smem = gl.allocate_shared_memory(
v_cache_ptr.dtype.element_ty, [cfg.BLOCK_N, cfg.HEAD_DIM], cfg.v_smem_layout
)
q = program.load_q()
m_i, l_i, acc = program.init_attention_state()
offs_m_q = program.q_start + gl.arange(
0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.qk_layout)
)
offs_n_q = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(0, cfg.qk_layout))
diag_row = program.prefix + offs_m_q
if IS_CAUSAL:
kv_end = min(program.cache_len, program.prefix + program.q_start + cfg.BLOCK_M)
else:
kv_end = program.cache_len
# Sliding window (inclusive-left, matches flash-attn window_size=(W, 0)):
# skip KV tiles entirely below the window's lower edge. The tile's top query
# row sits at absolute position pos_top; its window opens at
# pos_top - WINDOW_LEFT (that key is visible -> W + 1 keys total). The min()
# form clamps to 0 without the shadowed builtin max().
if cfg.WINDOW_LEFT >= 0:
pos_top = program.prefix + program.q_start
kv_start = pos_top - min(pos_top, cfg.WINDOW_LEFT)
kv_start = (kv_start // cfg.BLOCK_N) * cfg.BLOCK_N
else:
kv_start = 0
for start_n in range(kv_start, kv_end, cfg.BLOCK_N):
physical_page = program.load_page(start_n)
program.issue_load_k(physical_page, k_smem)
program.issue_load_v(physical_page, v_smem)
async_copy.wait_group(1)
k = program.shared_load_k(k_smem)
qk = program.compute_qk(q, k)
qk = program.apply_rel_bias(qk, start_n)
if cfg.WINDOW_LEFT >= 0:
# Window lower edge + cache bound always apply; causal upper edge only
# when IS_CAUSAL (independent layering keeps non-causal + window correct).
offs_n_abs = start_n + offs_n_q
mask = (offs_n_abs[None, :] >= diag_row[:, None] - cfg.WINDOW_LEFT) & (
offs_n_abs[None, :] < program.cache_len
)
if IS_CAUSAL:
mask &= offs_n_abs[None, :] <= diag_row[:, None]
qk = gl.where(mask, qk, -float("inf"))
elif IS_CAUSAL:
if start_n + cfg.BLOCK_N > program.prefix + program.q_start:
offs_n_abs = start_n + offs_n_q
mask = (offs_n_abs[None, :] <= diag_row[:, None]) & (
offs_n_abs[None, :] < program.cache_len
)
qk = gl.where(mask, qk, -float("inf"))
else:
if start_n + cfg.BLOCK_N > program.cache_len:
offs_n_abs = start_n + offs_n_q
qk = gl.where(
offs_n_abs[None, :] < program.cache_len, qk, -float("inf")
)
p, m_i, l_i, acc = program.softmax(qk, m_i, l_i, acc)
async_copy.wait_group(0)
v = program.shared_load_v(v_smem)
acc = program.compute_pv(p, v, acc)
denom = gl.where(l_i > 0.0, l_i, 1.0)
output = acc * (1.0 / denom)[:, None]
output = gl.convert_layout(output, cfg.store_layout)
program.store_output(output)
program.store_lse(l_i, m_i)
def gluon_rel_mha_extend_gfx950(
q: torch.Tensor,
cu_seqlens_q: torch.Tensor,
cu_seqlens_kv: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
page_table: torch.Tensor,
cache_seqlens: torch.Tensor,
window_left: int = -1,
return_lse: bool = False,
max_seqlen_q: int = 1,
max_seqlen_k: int = 1,
rel_logits: torch.Tensor | None = None,
softmax_scale: float | None = None,
q_scale: torch.Tensor | None = None,
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
out: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
assert rel_logits is not None
head_dim = q.shape[2]
n_heads = q.shape[1]
n_kv_heads = k_cache.shape[2]
page_size = k_cache.shape[1]
block_m, block_n, num_warps = _select_extend_tile(max_seqlen_q, page_size)
if softmax_scale is None:
softmax_scale = 1.0 / math.sqrt(head_dim)
sm_scale = softmax_scale * _INV_LN2_VALUE
rel_bias_qk_scale = 1.0 / softmax_scale
# max_seqlen_q must be >= the true max query length; extra tiles early-exit.
batch = cu_seqlens_q.shape[0] - 1
safe_max_q = max_seqlen_q if max_seqlen_q > 0 else 1
blocks_per_req = (safe_max_q + block_m - 1) // block_m
cu_q_i32 = cu_seqlens_q.to(torch.int32).contiguous()
cache_i32 = cache_seqlens.to(torch.int32).contiguous()
is_fp8 = q.dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
out_dtype = torch.bfloat16 if is_fp8 else q.dtype
if page_size not in (64, 128, 256):
raise ValueError(
"gfx950 Gluon relative-attention extend requires page size "
f"64, 128, or 256; got {page_size}"
)
output = (
torch.empty(q.shape, device=q.device, dtype=out_dtype) if out is None else out
)
if return_lse:
lse = torch.empty((q.shape[0], n_heads), device=q.device, dtype=torch.float32)
lse_arg = lse
else:
lse = None
lse_arg = q
grid = (blocks_per_req, batch, n_heads)
_rel_mha_extend_fp16[grid](
q,
rel_logits,
k_cache,
v_cache,
page_table,
output,
lse_arg,
cu_q_i32,
cache_i32,
q.stride(0),
q.stride(1),
q.stride(2),
k_cache.stride(0),
k_cache.stride(1),
k_cache.stride(2),
k_cache.stride(3),
v_cache.stride(0),
v_cache.stride(1),
v_cache.stride(2),
v_cache.stride(3),
sm_scale,
page_table.stride(0),
page_size,
n_heads,
n_kv_heads,
head_dim,
block_m,
block_n,
num_warps,
True,
return_lse,
window_left,
rel_logits.stride(0),
rel_logits.stride(1),
rel_logits.stride(2),
rel_logits.shape[2],
rel_bias_qk_scale,
is_fp8,
num_warps=num_warps,
)
if return_lse:
assert lse is not None
return output, lse
return output
+154
View File
@@ -0,0 +1,154 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from vllm.triton_utils import aggregate, gl, gluon, tl
_INV_LN2_VALUE = 1.4426950408889634
_INV_LN2 = tl.constexpr(_INV_LN2_VALUE)
_LN2_VALUE = 0.6931471805599453
_LN2 = tl.constexpr(_LN2_VALUE)
_PROPAGATE_NAN_ALL = gl.constexpr(tl.PropagateNan.ALL)
@gluon.jit
def maximum(a, b, propagate_nan: gl.constexpr = _PROPAGATE_NAN_ALL):
return gl.maximum(a, b, propagate_nan=propagate_nan)
@gluon.jit
def max(input, axis=None, keep_dims=False):
return gl.reduce(input, axis, maximum, keep_dims=keep_dims)
@gluon.constexpr_function
def attention_layouts(head_dim, block_n, is_fp8, dtype, num_warps, instr_shape):
mfma = gl.amd.AMDMFMALayout(
version=4,
instr_shape=instr_shape,
transposed=True,
warps_per_cta=[num_warps, 1],
)
qk_layout = mfma
pv_layout = mfma
# qk_kw is derived from a 128-bit load / dtype bitwidth; pv_kw is tuned.
qk_kw = 16 if is_fp8 else 8
pv_kw = 8 if is_fp8 else 4
q_layout = gl.DotOperandLayout(0, qk_layout, k_width=qk_kw)
k_layout = gl.DotOperandLayout(1, qk_layout, k_width=qk_kw)
p_layout = gl.DotOperandLayout(0, pv_layout, k_width=pv_kw)
v_layout = gl.DotOperandLayout(1, pv_layout, k_width=pv_kw)
# load_vec = elems/lane (dtype-dependent, == qk_kw); load_threads span HEAD_DIM.
load_vec = 16 if is_fp8 else 8
load_threads = head_dim // load_vec
load_layout = gl.BlockedLayout(
[1, load_vec], [64 // load_threads, load_threads], [num_warps, 1], [1, 0]
)
# store_vec is always 16-bit (128 / 16 = 8) regardless of input dtype.
store_vec = 8
store_threads = head_dim // store_vec
store_layout = gl.BlockedLayout(
[1, store_vec], [64 // store_threads, store_threads], [num_warps, 1], [1, 0]
)
# Take only the built-in's padding, not its swizzle: the swizzle scatters
# head_dim across banks, making the LDS write stride non-constant, so it can't
# lower through async_copy's affine [1, 0] load. Padding-only is affine and
# DMA-legal.
# TODO(perf): to also use the swizzle, co-design a matched load layout so the
# DMA stays legal.
k_api = gl.amd.cdna4.compute_efficient_padded_shared_layout(
k_layout, [block_n, head_dim], dtype, is_k_contig=True
)
v_api = gl.amd.cdna4.compute_efficient_padded_shared_layout(
v_layout, [block_n, head_dim], dtype, is_k_contig=False
)
assert k_api is not None and v_api is not None, (
"no CDNA4 padded shared layout for this operand/dtype"
)
k_pairs = list(k_api.interval_padding_pairs)
v_pairs = list(v_api.interval_padding_pairs)
assert len(k_pairs) == 1 and len(v_pairs) == 1, (
"expected a single interval padding pair from the built-in"
)
k_smem_layout = gl.PaddedSharedLayout.with_identity_for(
[[int(k_pairs[0][0]), int(k_pairs[0][1])]], [block_n, head_dim], [1, 0]
)
v_smem_layout = gl.PaddedSharedLayout.with_identity_for(
[[int(v_pairs[0][0]), int(v_pairs[0][1])]], [block_n, head_dim], [1, 0]
)
return (
qk_layout,
pv_layout,
q_layout,
k_layout,
p_layout,
v_layout,
load_layout,
store_layout,
k_smem_layout,
v_smem_layout,
)
@aggregate
class InputStrides:
stride_t: gl.constexpr
stride_h: gl.constexpr
stride_d: gl.constexpr
@gluon.constexpr_function
def __init__(self, stride_t, stride_h, stride_d):
self.stride_t = gl.constexpr(stride_t)
self.stride_h = gl.constexpr(stride_h)
self.stride_d = gl.constexpr(stride_d)
@gluon.jit
def offsets(self, token, head, dim):
return (token * self.stride_t + head * self.stride_h + dim * self.stride_d).to(
gl.int32
)
@aggregate
class PagedKVStrides:
stride_b: gl.constexpr
stride_p: gl.constexpr
stride_h: gl.constexpr
stride_d: gl.constexpr
@gluon.constexpr_function
def __init__(self, stride_b, stride_p, stride_h, stride_d):
self.stride_b = gl.constexpr(stride_b)
self.stride_p = gl.constexpr(stride_p)
self.stride_h = gl.constexpr(stride_h)
self.stride_d = gl.constexpr(stride_d)
@gluon.jit
def offsets(self, page, token, head, dim):
# KV pools can exceed the 32-bit buffer-offset range. Keep the page
# arithmetic in int64, matching the original kernel's large-cache path.
return (
page.to(gl.int64) * self.stride_b
+ token.to(gl.int64) * self.stride_p
+ head * self.stride_h
+ dim * self.stride_d
)
+766
View File
@@ -0,0 +1,766 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Deadlock-free fused RS + short-conv + AG + residual + RMSNorm.
The public integration surface is ``LamportRSConv.rs_sconv_ag_add_norm``.
Liveness
--------
Large grids are deliberately split at the two communication dependencies:
1. ``_publish_input_kernel`` only publishes rank partials.
2. ``_reduce_insert_kernel`` waits, reduces, and inserts the local shard.
3. ``_sconv_publish_kernel`` only computes and publishes the local result.
4. ``_gather_norm_kernel`` waits, gathers, and normalizes.
Without PDL, CUDA stream order completes a producer before its consumer. With
PDL, every producer CTA posts all peer stores before triggering its dependent,
and the consumer executes ``gdc_wait`` before polling. A consumer therefore
waits only for stores from a producer that is already running (or complete) on
another GPU. It never waits for another block in its own grid. Consequently
spinning consumers cannot occupy resources needed by any producer, and the
wait-for graph has no cycle. The proof is independent of grid size, block
dispatch order, and occupancy.
For one token, the first three phases use eight independent channel slices to
expose enough CTA parallelism for decode latency. Each slice has exclusive
ownership of its cache and Lamport columns; the gather/RMSNorm phase retains
one CTA per token so no cross-CTA reduction or completion counter is needed.
Immediate buffer reuse (including replay of a captured CUDA graph) is also
safe. Rank R cannot republish input for call n+1 until its gather for n has
finished; that gather waited for owner O's output, which O publishes only
after consuming R's input for n. Likewise, R cannot republish output for n+1
until its reduction for n+1 has observed destination D's input; D publishes
that input only after its gather consumed R's output for n. Thus every prior
read happens-before a same-slot rewrite. Three generations reduce incidental
coupling for ordinary launches, but correctness does not depend on rotation.
The payload itself is the Lamport flag. A 32-bit store publishes two bf16s
atomically; 0x80008000 (two negative zeroes) denotes an empty pair. Real
negative zeroes are changed to positive zero before publication. Consumers
use volatile 32-bit loads and restore the sentinel after consuming a slot.
"""
from __future__ import annotations
import os
import torch
from vllm.distributed import get_tp_group
from vllm.distributed.parallel_state import in_the_same_node_as
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
logger = init_logger(__name__)
_MAX_TOKENS = 16384
_EMPTY_PAIR = tl.constexpr(0x80008000)
@triton.jit
def _pack_bf16_pairs(values):
"""Pack bf16 values into atomic u32 pairs and reserve negative zero."""
lo, hi = tl.split(values.reshape([values.shape[0] // 2, 2]))
lo = lo.to(tl.uint16, bitcast=True)
hi = hi.to(tl.uint16, bitcast=True)
lo = tl.where(lo == 0x8000, 0, lo).to(tl.uint32)
hi = tl.where(hi == 0x8000, 0, hi).to(tl.uint32)
return lo | (hi << 16)
@triton.jit
def _unpack_bf16_pairs(values):
lo = (values & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True)
hi = (values >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True)
return tl.interleave(lo, hi)
@triton.jit
def _wait_pairs(ptr, offsets, mask):
values = tl.load(ptr + offsets, mask=mask, other=0, volatile=True)
while tl.max(tl.where(mask & (values == _EMPTY_PAIR), 1, 0)) != 0:
values = tl.load(ptr + offsets, mask=mask, other=0, volatile=True)
return values
@triton.jit
def _publish_input_kernel(
stage_ptr,
peer_ptrs,
peer_offset_u32,
stride_stage_t,
C: tl.constexpr,
CS: tl.constexpr,
CS_P2: tl.constexpr,
SPLITS: tl.constexpr,
RANK: tl.constexpr,
WORLD: tl.constexpr,
USE_PDL: tl.constexpr,
launch_pdl: tl.constexpr,
):
"""Publish this rank's full partial row into every shard owner's slots."""
token = tl.program_id(0).to(tl.int64)
split = tl.program_id(1).to(tl.int64)
CSS: tl.constexpr = CS // SPLITS
pair = tl.arange(0, CS_P2 // 2)
pair_mask = pair < CSS // 2
elem = tl.arange(0, CS_P2)
elem_mask = elem < CSS
ptrs = peer_ptrs.to(tl.pointer_type(tl.uint64))
# Address generation is independent of the preceding stream kernel. The
# acquire remains before stage/peer loads, which may consume its writes.
if USE_PDL:
tl.extra.cuda.gdc_wait()
for owner in tl.static_range(WORLD):
values = tl.load(
stage_ptr + token * stride_stage_t + owner * CS + split * CSS + elem,
mask=elem_mask,
other=0.0,
)
packed = _pack_bf16_pairs(values)
base = tl.load(ptrs + owner).to(tl.pointer_type(tl.uint32))
dst = (token * WORLD + RANK) * (CS // 2) + split * (CSS // 2) + pair
tl.store(base + peer_offset_u32 + dst, packed, mask=pair_mask)
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
@triton.jit
def _reduce_insert_kernel(
input_peer_ptrs,
input_peer_offset_u32,
cache_ptr,
slot_ptr,
stride_cache_block,
stride_cache_head,
stride_cache_token,
stride_cache_dim,
block_size,
C: tl.constexpr,
CS: tl.constexpr,
CS_P2: tl.constexpr,
SPLITS: tl.constexpr,
HEAD_SIZE: tl.constexpr,
CACHE_OFFSET: tl.constexpr,
RANK: tl.constexpr,
WORLD: tl.constexpr,
USE_PDL: tl.constexpr,
launch_pdl: tl.constexpr,
):
"""Consume all partials for this rank and insert the reduced cache row."""
token = tl.program_id(0).to(tl.int64)
split = tl.program_id(1).to(tl.int64)
CSS: tl.constexpr = CS // SPLITS
source = tl.arange(0, WORLD)
pair = tl.arange(0, CS_P2 // 2)
pair_mask = pair < CSS // 2
offsets = (
(token * WORLD + source)[:, None] * (CS // 2)
+ split * (CSS // 2)
+ pair[None, :]
)
mask = tl.full([WORLD], True, tl.int1)[:, None] & pair_mask[None, :]
input_ptrs = input_peer_ptrs.to(tl.pointer_type(tl.uint64))
input_u32 = tl.load(input_ptrs + RANK).to(tl.pointer_type(tl.uint32))
input_u32 += input_peer_offset_u32
# Slot metadata and the cache destination do not depend on input publish.
slot = tl.load(slot_ptr + token)
valid = slot >= 0
channel = tl.arange(0, CS_P2)
channel_mask = channel < CSS
global_channel = split * CSS + channel
head = tl.minimum(global_channel // HEAD_SIZE, CS // HEAD_SIZE - 1)
dim = CACHE_OFFSET + global_channel % HEAD_SIZE
safe_slot = tl.maximum(slot, 0).to(tl.int64)
dst = (
cache_ptr
+ (safe_slot // block_size) * stride_cache_block
+ head * stride_cache_head
+ (safe_slot % block_size) * stride_cache_token
+ dim * stride_cache_dim
)
if USE_PDL:
tl.extra.cuda.gdc_wait()
packed = _wait_pairs(input_u32, offsets, mask)
lo = (packed & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True)
hi = (packed >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True)
reduced = tl.interleave(
tl.sum(lo.to(tl.float32), axis=0).to(tl.bfloat16),
tl.sum(hi.to(tl.float32), axis=0).to(tl.bfloat16),
)
tl.store(
input_u32 + offsets,
tl.full([WORLD, CS_P2 // 2], _EMPTY_PAIR, tl.uint32),
mask=mask,
)
tl.store(dst, reduced, mask=valid & channel_mask)
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
@triton.jit
def _sconv_publish_kernel(
peer_ptrs,
peer_offset_u32,
residual_ptr,
weight_ptr,
cache_ptr,
position_ptr,
sequence_ptr,
slot_ptr,
block_table_ptr,
stride_residual_t,
stride_cache_block,
stride_cache_head,
stride_cache_token,
stride_cache_dim,
stride_block_table_r,
max_blocks,
block_size,
C: tl.constexpr,
CS: tl.constexpr,
CS_P2: tl.constexpr,
SPLITS: tl.constexpr,
HEAD_SIZE: tl.constexpr,
CACHE_OFFSET: tl.constexpr,
RANK: tl.constexpr,
WORLD: tl.constexpr,
WINDOW: tl.constexpr,
USE_PDL: tl.constexpr,
launch_pdl: tl.constexpr,
):
"""Compute this rank's shard, then publish it to every rank."""
tl.static_assert(WINDOW == 4)
token = tl.program_id(0).to(tl.int64)
split = tl.program_id(1).to(tl.int64)
CSS: tl.constexpr = CS // SPLITS
channel = tl.arange(0, CS_P2)
channel_mask = channel < CSS
global_channel = split * CSS + channel
head = tl.minimum(global_channel // HEAD_SIZE, CS // HEAD_SIZE - 1)
dim = CACHE_OFFSET + global_channel % HEAD_SIZE
slot = tl.load(slot_ptr + token)
valid = slot >= 0
position = tl.load(position_ptr + token)
sequence = tl.load(sequence_ptr + token)
safe_slot = tl.maximum(slot, 0).to(tl.int64)
own_ptr = (
cache_ptr
+ (safe_slot // block_size) * stride_cache_block
+ head * stride_cache_head
+ (safe_slot % block_size) * stride_cache_token
+ dim * stride_cache_dim
)
# These loads were made visible before reduce/insert could signal us and
# are independent of its cache store. Hoisting them gives PDL useful work
# to overlap while retaining the acquire before every cache read.
residual = tl.load(
residual_ptr + token * stride_residual_t + RANK * CS + global_channel,
mask=channel_mask,
other=0.0,
)
weight0 = tl.load(
weight_ptr + global_channel * WINDOW,
mask=channel_mask,
other=0.0,
).to(tl.float32)
weight1 = tl.load(
weight_ptr + global_channel * WINDOW + 1,
mask=channel_mask,
other=0.0,
).to(tl.float32)
weight2 = tl.load(
weight_ptr + global_channel * WINDOW + 2,
mask=channel_mask,
other=0.0,
).to(tl.float32)
weight3 = tl.load(
weight_ptr + global_channel * WINDOW + 3,
mask=channel_mask,
other=0.0,
).to(tl.float32)
ptrs = peer_ptrs.to(tl.pointer_type(tl.uint64))
if USE_PDL:
tl.extra.cuda.gdc_wait()
current = tl.load(own_ptr, mask=valid & channel_mask, other=0.0)
conv = tl.zeros([CS_P2], tl.float32)
for tap_idx in tl.static_range(WINDOW):
source_position = position - (WINDOW - 1) + tap_idx
take = valid & (source_position >= 0)
if tap_idx == WINDOW - 1:
value = tl.where(take, current.to(tl.float32), 0.0)
else:
safe_position = tl.maximum(source_position, 0)
logical_block = tl.minimum(safe_position // block_size, max_blocks - 1)
physical_block = tl.load(
block_table_ptr + sequence * stride_block_table_r + logical_block,
mask=take,
other=0,
).to(tl.int64)
source_ptr = (
cache_ptr
+ physical_block * stride_cache_block
+ head * stride_cache_head
+ (safe_position % block_size) * stride_cache_token
+ dim * stride_cache_dim
)
cached = tl.load(source_ptr, mask=take & channel_mask, other=0.0)
value = tl.where(take, cached.to(tl.float32), 0.0)
if tap_idx == 0:
weight = weight0
elif tap_idx == 1:
weight = weight1
elif tap_idx == 2:
weight = weight2
else:
weight = weight3
conv += value * weight
# Preserve both bf16 rounding points of the original sublayer.
short_conv_with_skip = (conv + current.to(tl.float32)).to(tl.bfloat16)
output = (residual.to(tl.float32) + short_conv_with_skip.to(tl.float32)).to(
tl.bfloat16
)
pair = tl.arange(0, CS_P2 // 2)
pair_mask = pair < CSS // 2
packed = _pack_bf16_pairs(output)
row_offset = token * (C // 2) + RANK * (CS // 2) + split * (CSS // 2) + pair
for destination in tl.static_range(WORLD):
base = tl.load(ptrs + destination).to(tl.pointer_type(tl.uint32))
tl.store(base + peer_offset_u32 + row_offset, packed, mask=pair_mask)
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
@triton.jit
def _gather_norm_kernel(
output_peer_ptrs,
output_peer_offset_u32,
norm_weight_ptr,
normed_ptr,
residual_out_ptr,
eps,
stride_output_t,
C: tl.constexpr,
C_P2: tl.constexpr,
RANK: tl.constexpr,
HAS_NORM: tl.constexpr,
USE_PDL: tl.constexpr,
launch_pdl: tl.constexpr,
):
"""Consume a complete row; one CTA owns all outputs for one token."""
token = tl.program_id(0).to(tl.int64)
pair = tl.arange(0, C_P2 // 2)
pair_mask = pair < C // 2
output_ptrs = output_peer_ptrs.to(tl.pointer_type(tl.uint64))
output_u32 = tl.load(output_ptrs + RANK).to(tl.pointer_type(tl.uint32))
output_u32 += output_peer_offset_u32
offsets = token * (C // 2) + pair
channel = tl.arange(0, C_P2)
channel_mask = channel < C
# Gamma is independent of the preceding sconv publication. Keep the
# acquire immediately before polling the Lamport output slots.
if HAS_NORM:
weight = tl.load(norm_weight_ptr + channel, mask=channel_mask, other=0.0)
if USE_PDL:
tl.extra.cuda.gdc_wait()
packed = _wait_pairs(output_u32, offsets, pair_mask)
row = _unpack_bf16_pairs(packed)
tl.store(
residual_out_ptr + token * stride_output_t + channel, row, mask=channel_mask
)
if HAS_NORM:
row_f32 = tl.where(channel_mask, row.to(tl.float32), 0.0)
inv_rms = tl.rsqrt(tl.sum(row_f32 * row_f32, axis=0) / C + eps)
tl.store(
normed_ptr + token * stride_output_t + channel,
(row_f32 * inv_rms * weight.to(tl.float32)).to(tl.bfloat16),
mask=channel_mask,
)
tl.store(
output_u32 + offsets,
tl.full([C_P2 // 2], _EMPTY_PAIR, tl.uint32),
mask=pair_mask,
)
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
@triton.jit
def _validate_lamport_init_kernel(
input_peer_ptrs,
output_peer_ptrs,
bad_ptr,
num_pairs,
RANK: tl.constexpr,
BLOCK: tl.constexpr,
):
"""Validate both complete local allocations through their fabric pointers."""
offsets = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < num_pairs
ptrs_in = input_peer_ptrs.to(tl.pointer_type(tl.uint64))
ptrs_out = output_peer_ptrs.to(tl.pointer_type(tl.uint64))
local_in = tl.load(ptrs_in + RANK).to(tl.pointer_type(tl.uint32))
local_out = tl.load(ptrs_out + RANK).to(tl.pointer_type(tl.uint32))
value_in = tl.load(local_in + offsets, mask=mask, other=_EMPTY_PAIR)
value_out = tl.load(local_out + offsets, mask=mask, other=_EMPTY_PAIR)
bad = tl.max(
tl.where(mask & ((value_in != _EMPTY_PAIR) | (value_out != _EMPTY_PAIR)), 1, 0)
)
if bad != 0:
tl.atomic_max(bad_ptr, 1)
class LamportRSConv:
"""Persistent symmetric buffers for one TP group."""
def _initialize_lamport_buffers(self) -> None:
"""Arm every slot and collectively verify the exact sentinel bits.
Do not replace this with a zero-fill: Lamport readiness distinguishes
the bf16 bit pattern for -0.0 (0x8000) from every published payload.
The validation is deliberately collective so one rank cannot enter the
first polling kernel while another rank still has an unarmed buffer.
"""
if not self.buf_in.is_contiguous() or not self.buf_out.is_contiguous():
raise RuntimeError("Lamport symmetric buffers must be contiguous")
if self.buf_in.numel() % 2 or self.buf_out.numel() % 2:
raise RuntimeError("Lamport buffers must contain whole uint32 pairs")
# Fill through int16 so each bf16 lane receives the exact -0.0 bits.
# This covers all three generations and all max-token slots, including
# slots that a smaller first invocation does not touch.
self.buf_in.view(torch.int16).fill_(-0x8000)
self.buf_out.view(torch.int16).fill_(-0x8000)
torch.accelerator.synchronize(self.device)
self.tp.barrier()
# Scan the complete local allocations. Since every rank performs the
# scan and participates in MAX, success proves every symmetric backing
# allocation was armed before any rank is allowed to use generation 0.
bad = torch.logical_or(
self.buf_in.view(torch.int16).ne(-0x8000).any(),
self.buf_out.view(torch.int16).ne(-0x8000).any(),
).to(dtype=torch.float32)
bad = self.tp.all_reduce(bad)
if int(bad.item()) != 0:
raise RuntimeError("Lamport sentinel initialization failed on a TP rank")
self.tp.barrier()
def _initialize_mnnvl_buffers(self) -> None:
"""Initialize and validate FlashInfer fabric-mapped Lamport storage."""
self._mnnvl_input_handle.lamport_initialize(self.rank, torch.bfloat16)
self._mnnvl_output_handle.lamport_initialize(self.rank, torch.bfloat16)
torch.accelerator.synchronize(self.device)
self.tp.barrier()
bad = torch.zeros((), dtype=torch.int32, device=self.device)
num_pairs = self.num_buffers * self.max_tokens * self.hidden_size // 2
_validate_lamport_init_kernel[(triton.cdiv(num_pairs, 256),)](
self.input_peer_ptrs,
self.output_peer_ptrs,
bad,
num_pairs,
RANK=self.rank,
BLOCK=256,
num_warps=4,
)
bad = self.tp.all_reduce(bad.to(torch.float32))
if int(bad.item()) != 0:
raise RuntimeError("MNNVL Lamport sentinel initialization failed")
self.tp.barrier()
def __init__(
self, hidden_size: int, window_size: int, max_tokens: int = _MAX_TOKENS
) -> None:
import torch.distributed._symmetric_memory as symm_mem
tp = get_tp_group()
self.tp = tp
self.group = tp.device_group
self.world_size = tp.world_size
self.rank = tp.rank_in_group
self.device = torch.device(tp.device)
if self.world_size not in (2, 4, 8):
raise ValueError(f"TP world size must be 2, 4, or 8, got {self.world_size}")
if hidden_size % (2 * self.world_size) != 0:
raise ValueError("hidden size must produce an even shard on every rank")
if window_size != 4:
raise ValueError(f"short-conv window size must be 4, got {window_size}")
if max_tokens < 1 or max_tokens > _MAX_TOKENS:
raise ValueError(f"max_tokens must be in [1, {_MAX_TOKENS}]")
is_cross_node = not all(in_the_same_node_as(tp.cpu_group))
self.hidden_size = hidden_size
self.window_size = window_size
self.max_tokens = max_tokens
self.shard_size = hidden_size // self.world_size
# Three generations follow FlashInfer's Lamport layout. A generation
# is reused only after two intervening collective calls.
self.num_buffers = 3
self.input_generation_bytes = max_tokens * hidden_size * 2
self.output_generation_bytes = max_tokens * hidden_size * 2
self.use_pdl = torch.cuda.get_device_capability(self.device)[0] >= 9
if is_cross_node:
try:
from flashinfer.comm.mnnvl import (
McastGPUBuffer,
TorchDistBackend,
is_mnnvl_fabric_supported,
)
except ImportError as error:
raise RuntimeError(
"cross-node TP requires FlashInfer MNNVL support"
) from error
local_supported = int(
is_mnnvl_fabric_supported(torch.accelerator.current_device_index())
)
unsupported = torch.tensor(
1 - local_supported, dtype=torch.float32, device=self.device
)
unsupported = tp.all_reduce(unsupported)
if int(unsupported.item()) != 0:
raise RuntimeError("cross-node TP is supported only on MNNVL fabric")
comm_backend = TorchDistBackend(self.group)
allocation_bytes = self.num_buffers * max_tokens * hidden_size * 2
self._mnnvl_input_handle = McastGPUBuffer(
allocation_bytes,
self.world_size,
self.rank,
self.device,
comm_backend,
)
self._mnnvl_output_handle = McastGPUBuffer(
allocation_bytes,
self.world_size,
self.rank,
self.device,
comm_backend,
)
self.input_peer_ptrs = self._mnnvl_input_handle.get_buffer_ptrs_dev()
self.output_peer_ptrs = self._mnnvl_output_handle.get_buffer_ptrs_dev()
self._initialize_mnnvl_buffers()
logger.info("using FlashInfer fabric-mapped MNNVL Lamport buffers")
else:
self.buf_in = symm_mem.empty(
self.num_buffers,
max_tokens,
self.world_size,
self.shard_size,
dtype=torch.bfloat16,
device=self.device,
)
self.buf_out = symm_mem.empty(
self.num_buffers,
max_tokens,
hidden_size,
dtype=torch.bfloat16,
device=self.device,
)
group_name = self.group.group_name
input_handle = symm_mem.rendezvous(self.buf_in, group_name)
output_handle = symm_mem.rendezvous(self.buf_out, group_name)
self.input_peer_ptrs = input_handle.buffer_ptrs_dev
self.output_peer_ptrs = output_handle.buffer_ptrs_dev
self._input_handle = input_handle
self._output_handle = output_handle
self._initialize_lamport_buffers()
self.generation = 0
def usable(self, num_tokens: int) -> bool:
return 0 < num_tokens <= self.max_tokens
def rs_sconv_ag_add_norm(
self,
input_tensor: torch.Tensor,
residual: torch.Tensor,
conv_weight: torch.Tensor,
norm_weight: torch.Tensor | None,
eps: float,
cache: torch.Tensor,
positions: torch.Tensor,
block_table: torch.Tensor,
seq_idx: torch.Tensor,
slot_mapping: torch.Tensor,
off_s: int,
ws: int,
block_size: int,
) -> tuple[torch.Tensor | None, torch.Tensor]:
"""Return ``(normed | None, new_residual)``, both shaped ``[T, 6144]``."""
tokens, hidden_size = residual.shape
if not self.usable(tokens):
raise ValueError(f"num_tokens must be in [1, {self.max_tokens}]")
if hidden_size != self.hidden_size or residual.dtype != torch.bfloat16:
raise ValueError("residual must be bf16 [T, 6144]")
if (
input_tensor.shape != residual.shape
or input_tensor.dtype != torch.bfloat16
or input_tensor.stride(1) != 1
):
raise ValueError("input_tensor must be channel-contiguous bf16 [T, 6144]")
shard_size = hidden_size // self.world_size
if conv_weight.shape != (shard_size, self.window_size):
raise ValueError(
f"conv_weight must have shape [{shard_size}, {self.window_size}]"
)
if (
conv_weight.dtype != torch.bfloat16
or conv_weight.stride(0) != self.window_size
):
raise ValueError("conv_weight must be contiguous bf16")
if norm_weight is not None and (
norm_weight.shape != (hidden_size,) or norm_weight.dtype != torch.bfloat16
):
raise ValueError("norm_weight must be bf16 [6144] or None")
if cache.dtype != torch.bfloat16 or cache.ndim != 4:
raise ValueError("cache must be a 4-D bf16 tensor")
if shard_size % ws != 0 or cache.shape[1] != shard_size // ws:
raise ValueError("cache head layout is inconsistent with ws")
if off_s < 0 or off_s + ws > cache.shape[3]:
raise ValueError("cache channel offset is out of bounds")
index = self.generation
input_offset = index * self.input_generation_bytes // 4
output_offset = index * self.output_generation_bytes // 4
normed = torch.empty_like(residual) if norm_weight is not None else None
residual_out = torch.empty_like(residual)
phase_splits = 8 if tokens == 1 and shard_size % 8 == 0 else 1
phase_tile_p2 = triton.next_power_of_2(shard_size // phase_splits)
phase_grid = (tokens, phase_splits)
# Wide CTAs win before the grid saturates; smaller CTAs reduce register
# pressure once high-throughput batches provide enough parallelism.
if 128 <= tokens <= 2048:
phase_warps = 16
elif tokens > 2048:
phase_warps = 8
else:
phase_warps = 4
gather_warps = 4 if tokens >= 256 else 8
_publish_input_kernel[phase_grid](
input_tensor,
self.input_peer_ptrs,
input_offset,
input_tensor.stride(0),
C=hidden_size,
CS=shard_size,
CS_P2=phase_tile_p2,
SPLITS=phase_splits,
RANK=self.rank,
WORLD=self.world_size,
USE_PDL=self.use_pdl,
launch_pdl=self.use_pdl,
num_warps=phase_warps,
)
_reduce_insert_kernel[phase_grid](
self.input_peer_ptrs,
input_offset,
cache,
slot_mapping,
cache.stride(0),
cache.stride(1),
cache.stride(2),
cache.stride(3),
block_size,
C=hidden_size,
CS=shard_size,
CS_P2=phase_tile_p2,
SPLITS=phase_splits,
HEAD_SIZE=ws,
CACHE_OFFSET=off_s,
RANK=self.rank,
WORLD=self.world_size,
USE_PDL=self.use_pdl,
launch_pdl=self.use_pdl,
num_warps=phase_warps,
)
_sconv_publish_kernel[phase_grid](
self.output_peer_ptrs,
output_offset,
residual,
conv_weight,
cache,
positions,
seq_idx,
slot_mapping,
block_table,
residual.stride(0),
cache.stride(0),
cache.stride(1),
cache.stride(2),
cache.stride(3),
block_table.stride(0),
block_table.shape[1],
block_size,
C=hidden_size,
CS=shard_size,
CS_P2=phase_tile_p2,
SPLITS=phase_splits,
HEAD_SIZE=ws,
CACHE_OFFSET=off_s,
RANK=self.rank,
WORLD=self.world_size,
WINDOW=self.window_size,
USE_PDL=self.use_pdl,
launch_pdl=self.use_pdl,
num_warps=phase_warps,
)
_gather_norm_kernel[(tokens,)](
self.output_peer_ptrs,
output_offset,
norm_weight if norm_weight is not None else residual,
normed if normed is not None else residual_out,
residual_out,
eps,
residual.stride(0),
C=hidden_size,
C_P2=triton.next_power_of_2(hidden_size),
RANK=self.rank,
HAS_NORM=norm_weight is not None,
USE_PDL=self.use_pdl,
launch_pdl=self.use_pdl,
num_warps=gather_warps,
)
self.generation = (index + 1) % self.num_buffers
return normed, residual_out
_STATE: LamportRSConv | None = None
_STATE_FAILED = False
def initialize_lamport_rs_conv(
hidden_size: int, window_size: int, max_num_batched_tokens: int
) -> None:
"""Collectively initialize the TP-group state during model construction."""
global _STATE, _STATE_FAILED
if _STATE is not None:
if _STATE.hidden_size != hidden_size or _STATE.window_size != window_size:
raise RuntimeError("all Lamport users must share hidden and window sizes")
return
if _STATE_FAILED or os.environ.get("LAMPORT_RS_SCONV", "1") == "0":
return
try:
max_tokens = min(_MAX_TOKENS, max_num_batched_tokens)
_STATE = LamportRSConv(hidden_size, window_size, max_tokens=max_tokens)
except Exception:
_STATE_FAILED = True
logger.exception("fused collective unavailable; use the NCCL fallback")
def get_lamport_rs_conv(hidden_size: int, window_size: int) -> LamportRSConv | None:
"""Return the state initialized with the model, or ``None`` for fallback."""
if _STATE is not None and (
_STATE.hidden_size != hidden_size or _STATE.window_size != window_size
):
raise RuntimeError("all Lamport users must share hidden and window sizes")
return _STATE
+190
View File
@@ -0,0 +1,190 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused CUDA kernels for the Inkling vision/audio towers.
Both kernels keep the reference paths' fp32 accumulation and per-op bf16
rounding points (native ``rms_norm`` / ``F.gelu``); outputs are frequently
bit-identical and otherwise differ by 1-2 bf16 ulps from reduction-order
(real-checkpoint-weight cosine vs reference > 0.9999998).
"""
from __future__ import annotations
import torch
from vllm.triton_utils import tl, tldevice, triton
from .norm import _get_num_warps_from_block_size
@triton.jit
def _dmel_embed_sum_norm_kernel(
idx_ptr, # [T, NB] int32 dMel bin indices (values in [0, VOCAB))
w_ptr, # [NB * VOCAB, D] bf16 embedding table
norm_w_ptr, # [D] (unused if HAS_NORM=False)
out_ptr, # [T, D] bf16
eps,
D,
stride_idx_t,
NB: tl.constexpr,
VOCAB: tl.constexpr,
D_P2: tl.constexpr,
HAS_NORM: tl.constexpr,
):
t = tl.program_id(0).to(tl.int64)
offs = tl.arange(0, D_P2)
mask = offs < D
# One embedding row per mel bin (bin b uses table rows [b*VOCAB, (b+1)*VOCAB)),
# summed in fp32 (matches torch's fp32-accumulated bf16 .sum()).
acc = tl.zeros([D_P2], dtype=tl.float32)
for b in tl.static_range(NB):
v = tl.load(idx_ptr + t * stride_idx_t + b)
row = (b * VOCAB + v).to(tl.int64)
acc += tl.load(w_ptr + row * D + offs, mask=mask, other=0.0).to(tl.float32)
h = acc.to(tl.bfloat16)
if HAS_NORM:
# Match ir.ops.rms_norm: fp32 variance/normalize, then a single-rounded
# bf16 multiply with the bf16 weight.
x32 = h.to(tl.float32)
var = tl.sum(tl.where(mask, x32 * x32, 0.0), axis=0) / D
xn = (x32 * tl.math.rsqrt(var + eps)).to(tl.bfloat16)
w = tl.load(norm_w_ptr + offs, mask=mask, other=0.0)
h = xn * w
tl.store(out_ptr + t * D + offs, h, mask=mask)
def dmel_embed_sum_norm(
dmel_idx: torch.Tensor, # [T, NB] int32
weight: torch.Tensor, # [NB * VOCAB, D] bf16
norm_weight: torch.Tensor | None,
eps: float,
) -> torch.Tensor:
"""``rmsnorm(sum_b weight[b * VOCAB + idx[:, b]])`` in one launch (no
[T, NB, D] intermediate)."""
T, nb = dmel_idx.shape
D = weight.shape[1]
assert weight.shape[0] % nb == 0
vocab = weight.shape[0] // nb
out = torch.empty((T, D), dtype=weight.dtype, device=weight.device)
if T == 0:
return out
d_p2 = triton.next_power_of_2(D)
_dmel_embed_sum_norm_kernel[(T,)](
dmel_idx,
weight,
norm_weight if norm_weight is not None else weight,
out,
eps,
D,
dmel_idx.stride(0),
NB=nb,
VOCAB=vocab,
D_P2=d_p2,
HAS_NORM=norm_weight is not None,
# Swept on GB200: 8 warps beats the block-size heuristic's 16 by ~4%
# at large T (the 80-row gather chain is latency- not lane-bound).
num_warps=8,
)
return out
@triton.jit
def _rmsnorm_gelu_kernel(
x_ptr, # [R, D] bf16
w_ptr, # [D]
out_ptr, # [R, D] bf16 (or the folded layout when FOLD)
eps,
R,
D,
D_P2: tl.constexpr,
BLOCK_M: tl.constexpr, # rows per block (>1 for small D)
HAS_GELU: tl.constexpr,
FOLD: tl.constexpr,
# fold geometry: input rows index [N, T, H, W]; the store scatters each
# row to (out_row, slot) of fold_timespace_to_depth's output layout.
FT: tl.constexpr,
FH: tl.constexpr,
FW: tl.constexpr,
TF: tl.constexpr,
HF: tl.constexpr,
):
pid = tl.program_id(0).to(tl.int64)
rows = pid * BLOCK_M + tl.arange(0, BLOCK_M)
rmask = rows < R
offs = tl.arange(0, D_P2)
mask = rmask[:, None] & (offs < D)[None, :]
x32 = tl.load(x_ptr + rows[:, None] * D + offs[None, :], mask=mask, other=0.0).to(
tl.float32
)
var = tl.sum(x32 * x32, axis=1) / D
xn = (x32 * tl.math.rsqrt(var + eps)[:, None]).to(tl.bfloat16)
w = tl.load(w_ptr + offs, mask=offs < D, other=0.0)
h = xn * w[None, :] # bf16 multiply, matching ir.ops.rms_norm
if HAS_GELU:
# Exact (erf) GELU on the bf16-rounded norm output, fp32 math,
# matching F.gelu's opmath on a bf16 tensor.
g32 = h.to(tl.float32)
h = (0.5 * g32 * (1.0 + tldevice.erf(g32 * 0.7071067811865476))).to(tl.bfloat16)
if FOLD:
# Store directly into the next layer's folded layout (a pure
# permutation — replaces the separate fold copy pass).
t = (rows // (FH * FW)) % FT
hh = (rows // FW) % FH
ww = rows % FW
n = rows // (FT * FH * FW)
slot = ((t % TF) * HF + hh % HF) * HF + ww % HF
out_row = ((n * (FT // TF) + t // TF) * (FH // HF) + hh // HF) * (
FW // HF
) + ww // HF
base = (out_row * (TF * HF * HF) + slot) * D
tl.store(out_ptr + base[:, None] + offs[None, :], h, mask=mask)
else:
tl.store(out_ptr + rows[:, None] * D + offs[None, :], h, mask=mask)
def rmsnorm_gelu(
x: torch.Tensor, # [..., D] bf16 contiguous
weight: torch.Tensor,
eps: float,
gelu: bool = True,
fold: tuple[int, int] | None = None, # (t_fold, hw_fold) of the NEXT fold
) -> torch.Tensor:
"""Fused ``gelu(rmsnorm(x))`` (or plain rmsnorm); multiple rows per block
when D is small. With ``fold``, x must be [N, T, H, W, D] and the output
comes back as ``fold_timespace_to_depth(result, *fold)``."""
D = x.shape[-1]
flat = x.reshape(-1, D)
assert flat.stride(1) == 1 and flat.stride(0) == D
R = flat.shape[0]
if fold is None:
out = torch.empty_like(flat)
ft = fh = fw = tf = hf = 1
out_shape = x.shape
else:
tf, hf = fold
N, ft, fh, fw, _ = x.shape
out_shape = (N, ft // tf, fh // hf, fw // hf, tf * hf * hf * D)
out = torch.empty(out_shape, dtype=x.dtype, device=x.device)
if R == 0:
return out.reshape(out_shape)
d_p2 = triton.next_power_of_2(D)
block_m = max(1, 4096 // d_p2)
_rmsnorm_gelu_kernel[(triton.cdiv(R, block_m),)](
flat,
weight,
out,
eps,
R,
D,
D_P2=d_p2,
BLOCK_M=block_m,
HAS_GELU=gelu,
FOLD=fold is not None,
FT=ft,
FH=fh,
FW=fw,
TF=tf,
HF=hf,
num_warps=_get_num_warps_from_block_size(d_p2 * block_m),
)
return out.reshape(out_shape)
+414
View File
@@ -0,0 +1,414 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from functools import lru_cache
import torch
from vllm.triton_utils import tl, triton
_MAX_FUSED_SIZE = 65536
def _get_num_warps_from_block_size(block_size: int) -> int:
if block_size >= 32768:
return 32
if block_size >= 8192:
return 16
if block_size >= 2048:
return 8
return 4
def _largest_power_of_2(n: int) -> int:
assert n > 0, f"{n=}"
return 1 << (n.bit_length() - 1)
@lru_cache(maxsize=128)
def _get_grid_size_for_mem_bw_kernel(device: torch.device, factor: int = 8) -> int:
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
return _largest_power_of_2(num_sms) * factor
@triton.jit
def _rmsnorm_fwd_kernel(
x_ptr,
weight_ptr,
y_ptr,
rstd_ptr,
eps,
x_stride_0,
y_stride_0,
n_cols,
block_size_n: tl.constexpr,
):
pid_m = tl.program_id(0).to(tl.int64)
offs_n = tl.arange(0, block_size_n)
mask_n = offs_n < n_cols
weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
x = tl.load(x_ptr + pid_m * x_stride_0 + offs_n, mask=mask_n, other=0.0).to(
tl.float32
)
row_var = tl.sum(x * x, axis=0) / n_cols
rstd = tl.math.rsqrt(row_var + eps)
tl.store(rstd_ptr + pid_m, rstd)
y = x * rstd * weight
tl.store(y_ptr + pid_m * y_stride_0 + offs_n, y, mask=mask_n)
@triton.jit(do_not_specialize=["n_rows"])
def _rmsnorm_fwd_kernel_block_m(
x_ptr,
weight_ptr,
y_ptr,
rstd_ptr,
eps,
x_stride_0,
y_stride_0,
n_rows,
n_cols,
block_size_m: tl.constexpr,
block_size_n: tl.constexpr,
):
pid_m = tl.program_id(0).to(tl.int64)
offs_n = tl.arange(0, block_size_n)
mask_n = offs_n < n_cols
num_blocks_m = tl.cdiv(n_rows, block_size_m)
blocks_per_pid = tl.cdiv(num_blocks_m, tl.num_programs(0))
block_id_start = pid_m * blocks_per_pid
block_id_end = min(block_id_start + blocks_per_pid, num_blocks_m)
weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
for block_id in range(block_id_start, block_id_end):
offs_m = block_id * block_size_m + tl.arange(0, block_size_m)
mask_m = offs_m < n_rows
mask_mn = mask_m[:, None] & mask_n[None, :]
x = tl.load(
x_ptr + offs_m[:, None] * x_stride_0 + offs_n[None, :],
mask=mask_mn,
other=0.0,
).to(tl.float32)
row_var = tl.sum(x * x, axis=1) / n_cols
rstd = tl.math.rsqrt(row_var + eps)
tl.store(rstd_ptr + offs_m, rstd, mask=mask_m)
y = x * rstd[:, None] * weight
tl.store(
y_ptr + offs_m[:, None] * y_stride_0 + offs_n[None, :],
y,
mask=mask_mn,
)
@triton.jit
def _add_rmsnorm_fwd_kernel(
res_ptr, # [T, N] residual (read)
delta_ptr, # [T, N] delta to add (read)
weight_ptr,
y_ptr, # [T, N] normed output
res_out_ptr, # [T, N] updated residual output
eps,
res_stride_0,
delta_stride_0,
y_stride_0,
res_out_stride_0,
n_cols,
block_size_n: tl.constexpr,
):
pid_m = tl.program_id(0).to(tl.int64)
offs_n = tl.arange(0, block_size_n)
mask_n = offs_n < n_cols
weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
r = tl.load(res_ptr + pid_m * res_stride_0 + offs_n, mask=mask_n, other=0.0).to(
tl.float32
)
d = tl.load(delta_ptr + pid_m * delta_stride_0 + offs_n, mask=mask_n, other=0.0).to(
tl.float32
)
# Round the sum to the residual dtype first (matches the eager
# `residual + delta` then rmsnorm-on-bf16 sequence bit-for-bit).
s = (r + d).to(res_out_ptr.dtype.element_ty)
tl.store(res_out_ptr + pid_m * res_out_stride_0 + offs_n, s, mask=mask_n)
x = s.to(tl.float32)
row_var = tl.sum(x * x, axis=0) / n_cols
rstd = tl.math.rsqrt(row_var + eps)
y = x * rstd * weight
tl.store(y_ptr + pid_m * y_stride_0 + offs_n, y, mask=mask_n)
def add_rmsnorm(
residual: torch.Tensor,
delta: torch.Tensor,
weight: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused ``res = residual + delta; y = rmsnorm(res)``.
Returns ``(y, res)``; both are fresh tensors (cudagraph-friendly, no
in-place update of the inputs).
"""
assert residual.ndim == 2 and delta.ndim == 2, (residual.shape, delta.shape)
n_rows, n_cols = residual.shape
assert weight.shape[0] == n_cols
y = torch.empty_like(residual)
res_out = torch.empty_like(residual)
if n_rows == 0:
return y, res_out
block_size_n = triton.next_power_of_2(n_cols)
max_block_size_n = _MAX_FUSED_SIZE // residual.element_size()
if max_block_size_n < block_size_n:
raise RuntimeError(f"Large {n_cols=} is not supported")
num_warps = _get_num_warps_from_block_size(block_size_n)
_add_rmsnorm_fwd_kernel[(n_rows,)](
residual,
delta,
weight,
y,
res_out,
eps,
residual.stride(0),
delta.stride(0),
y.stride(0),
res_out.stride(0),
n_cols,
block_size_n,
num_warps=num_warps,
)
return y, res_out
@triton.jit
def _embed_rmsnorm_kernel(
ids_ptr, # [T] token ids
table_ptr, # [V, N] embedding table
weight_ptr, # [N] (HAS_NORM only)
chain_weight_ptr, # [N] (HAS_CHAIN only)
out_ptr, # [T, N] rmsnorm(table[ids], weight)
chain_out_ptr, # [T, N] rmsnorm(out, chain_weight) (HAS_CHAIN only)
eps,
table_stride_0,
n_cols,
block_size_n: tl.constexpr,
HAS_NORM: tl.constexpr,
HAS_CHAIN: tl.constexpr,
):
pid_m = tl.program_id(0).to(tl.int64)
offs_n = tl.arange(0, block_size_n)
mask_n = offs_n < n_cols
row = tl.load(ids_ptr + pid_m).to(tl.int64)
x = tl.load(table_ptr + row * table_stride_0 + offs_n, mask=mask_n, other=0.0)
if HAS_NORM:
xf = x.to(tl.float32)
rstd = tl.math.rsqrt(tl.sum(xf * xf, axis=0) / n_cols + eps)
w = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
# Round to the output dtype so the chained norm is bit-exact vs the
# unfused pair (which stores bf16 in between).
x = (xf * rstd * w).to(out_ptr.dtype.element_ty)
tl.store(out_ptr + pid_m * n_cols + offs_n, x, mask=mask_n)
if HAS_CHAIN:
xf = x.to(tl.float32)
rstd = tl.math.rsqrt(tl.sum(xf * xf, axis=0) / n_cols + eps)
w = tl.load(chain_weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
tl.store(
chain_out_ptr + pid_m * n_cols + offs_n,
(xf * rstd * w).to(chain_out_ptr.dtype.element_ty),
mask=mask_n,
)
def embed_rmsnorm(
input_ids: torch.Tensor,
embed_table: torch.Tensor,
weight: torch.Tensor | None,
eps: float,
chain_weight: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Fused ``rmsnorm(embed_table[input_ids], weight)`` row gather + norm.
Requires the full vocab on-rank (replicated or tp_size == 1).
``weight=None`` skips the norm (``use_embed_norm=False``), leaving a pure
embedding-table gather.
``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
first decoder layer's pre-attention norm) as a second output, still one
launch. Bit-exact vs the unfused module sequence."""
ids = input_ids.view(-1)
(T,) = ids.shape
n = embed_table.shape[1]
out = torch.empty(
(*input_ids.shape, n), dtype=embed_table.dtype, device=embed_table.device
)
chain_out = torch.empty_like(out) if chain_weight is not None else None
if T > 0:
block_size_n = triton.next_power_of_2(n)
_embed_rmsnorm_kernel[(T,)](
ids,
embed_table,
weight if weight is not None else embed_table,
chain_weight if chain_weight is not None else embed_table,
out,
chain_out if chain_out is not None else out,
eps,
embed_table.stride(0),
n,
block_size_n,
HAS_NORM=weight is not None,
HAS_CHAIN=chain_weight is not None,
num_warps=_get_num_warps_from_block_size(block_size_n),
)
if chain_out is not None:
return out, chain_out
return out
@triton.jit
def _embed_dual_rmsnorm_cat_kernel(
hidden_ptr, # [T, N]
emb_ptr, # [T, N] embeddings, or the [V, N] embedding table when GATHER
ids_ptr, # [T] token ids (GATHER only)
w_hidden_ptr, # [N]
w_pre_ptr, # [N] chained pre-norm on the embed side (HAS_PRE_NORM only)
w_embed_ptr, # [N]
out_ptr, # [T, 2N]: [rmsnorm(hidden) | rmsnorm(rmsnorm?(emb))]
eps,
hidden_stride_0,
emb_stride_0,
n_cols,
block_size_n: tl.constexpr,
GATHER: tl.constexpr,
HAS_PRE_NORM: tl.constexpr,
):
pid_m = tl.program_id(0).to(tl.int64)
which = tl.program_id(1) # 0 -> hidden into cols [0, N); 1 -> emb into [N, 2N)
offs_n = tl.arange(0, block_size_n)
mask_n = offs_n < n_cols
if which == 0:
x = tl.load(
hidden_ptr + pid_m * hidden_stride_0 + offs_n, mask=mask_n, other=0.0
).to(tl.float32)
w = tl.load(w_hidden_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
else:
row = tl.load(ids_ptr + pid_m).to(tl.int64) if GATHER else pid_m
x = tl.load(emb_ptr + row * emb_stride_0 + offs_n, mask=mask_n, other=0.0).to(
tl.float32
)
if HAS_PRE_NORM:
w_pre = tl.load(w_pre_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / n_cols + eps)
# Round-trip through the output dtype so the chained norm is
# bit-exact vs the unfused pair (which stores bf16 in between).
x = (x * rstd * w_pre).to(out_ptr.dtype.element_ty).to(tl.float32)
w = tl.load(w_embed_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / n_cols + eps)
tl.store(
out_ptr + pid_m * (2 * n_cols) + which * n_cols + offs_n,
(x * rstd * w).to(out_ptr.dtype.element_ty),
mask=mask_n,
)
def embed_dual_rmsnorm_cat(
hidden: torch.Tensor,
hidden_weight: torch.Tensor,
embed_weight: torch.Tensor,
eps: float,
*,
embeds: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
embed_table: torch.Tensor | None = None,
pre_norm_weight: torch.Tensor | None = None,
) -> torch.Tensor:
"""The MTP depth-layer input in one launch:
``cat([rmsnorm(hidden, w_h), rmsnorm(pre?(emb), w_e)], -1)``.
The embed side is either a fused row gather ``embed_table[input_ids]``
(draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged
multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the
backbone embed_norm in front of the depth embed_norm (bit-exact vs the
unfused sequence). The concat copies collapse into direct writes."""
T, n = hidden.shape
if embeds is not None:
assert embeds.shape == hidden.shape
src, ids, src_stride = embeds, embeds, embeds.stride(0)
gather = False
else:
assert input_ids is not None and embed_table is not None
assert input_ids.shape == (T,) and embed_table.shape[1] == n
src, ids, src_stride = embed_table, input_ids, embed_table.stride(0)
gather = True
out = torch.empty((T, 2 * n), dtype=hidden.dtype, device=hidden.device)
if T == 0:
return out
block_size_n = triton.next_power_of_2(n)
_embed_dual_rmsnorm_cat_kernel[(T, 2)](
hidden,
src,
ids,
hidden_weight,
pre_norm_weight if pre_norm_weight is not None else embed_weight,
embed_weight,
out,
eps,
hidden.stride(0),
src_stride,
n,
block_size_n,
GATHER=gather,
HAS_PRE_NORM=pre_norm_weight is not None,
num_warps=_get_num_warps_from_block_size(block_size_n),
)
return out
def rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
assert x.ndim == 2, f"{x.shape=}"
assert weight.ndim == 1, f"{weight.shape=}"
n_rows, n_cols = x.shape
assert weight.shape[0] == n_cols, f"{weight.shape=} {x.shape=}"
y = torch.empty_like(x)
rstd = torch.empty((n_rows,), dtype=torch.float32, device=x.device)
block_size_n = triton.next_power_of_2(n_cols)
max_block_size_n = _MAX_FUSED_SIZE // x.element_size()
if max_block_size_n < block_size_n:
raise RuntimeError(f"Large {n_cols=} is not supported")
block_size_m = max(1, 4096 // block_size_n)
num_warps = _get_num_warps_from_block_size(block_size_n)
if block_size_m == 1:
_rmsnorm_fwd_kernel[(n_rows,)](
x,
weight,
y,
rstd,
eps,
x.stride(0),
y.stride(0),
n_cols,
block_size_n,
num_warps=num_warps,
)
else:
grid_size = _get_grid_size_for_mem_bw_kernel(x.device)
_rmsnorm_fwd_kernel_block_m[(grid_size,)](
x,
weight,
y,
rstd,
eps,
x.stride(0),
y.stride(0),
n_rows,
n_cols,
block_size_m,
block_size_n,
num_warps=num_warps,
)
return y
+918
View File
@@ -0,0 +1,918 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import aux_stream
LOW_BLOCK_M = 32
LOW_BLOCK_N = 64
LOW_NUM_WARPS = 4
THROUGHPUT_BLOCK_M = 32
THROUGHPUT_BLOCK_N = 128
THROUGHPUT_GROUP_M = 2
THROUGHPUT_NUM_WARPS = 4
SMALL_TOKEN_THRESHOLD = 128
SMALL_NUM_WARPS = 2
Q_BLOCK_ROWS = 8
Q_NUM_WARPS = 2
KV_BLOCK_ROWS = 4
KV_NUM_WARPS = 2
@triton.jit(do_not_specialize=["rows"])
def _rel_proj_low_latency_kernel(
qkvr_ptr,
rel_proj_ptr,
rel_out_ptr,
log_scaling_ptr,
rows,
stride_x_t,
R_OFFSET: tl.constexpr,
NUM_Q_HEADS: tl.constexpr,
REL_EXTENT: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
APPLY_LOG_SCALING: tl.constexpr,
):
row = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
col = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
inner = tl.arange(0, 16)
token = row // NUM_Q_HEADS
head = row % NUM_Q_HEADS
relative = tl.load(
qkvr_ptr
+ token[:, None] * stride_x_t
+ R_OFFSET
+ head[:, None] * 16
+ inner[None, :],
mask=row[:, None] < rows,
other=0.0,
)
projection = tl.load(
rel_proj_ptr + inner[:, None] * REL_EXTENT + col[None, :],
mask=col[None, :] < REL_EXTENT,
other=0.0,
)
values = tl.dot(relative, projection, out_dtype=tl.float32).to(
rel_out_ptr.dtype.element_ty
)
values = values.to(tl.float32)
if APPLY_LOG_SCALING:
values *= tl.load(log_scaling_ptr + token, mask=row < rows, other=1.0)[:, None]
tl.store(
rel_out_ptr + row[:, None] * REL_EXTENT + col[None, :],
values.to(rel_out_ptr.dtype.element_ty),
mask=(row[:, None] < rows) & (col[None, :] < REL_EXTENT),
)
@triton.jit(do_not_specialize=["rows"])
def _rel_proj_throughput_kernel(
qkvr_ptr,
rel_proj_ptr,
rel_out_ptr,
log_scaling_ptr,
rows,
stride_x_t,
R_OFFSET: tl.constexpr,
NUM_Q_HEADS: tl.constexpr,
REL_EXTENT: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
GROUP_M: tl.constexpr,
APPLY_LOG_SCALING: tl.constexpr,
):
row_group = tl.program_id(0)
col = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
inner = tl.arange(0, 16)
projection = tl.load(
rel_proj_ptr + inner[:, None] * REL_EXTENT + col[None, :],
mask=col[None, :] < REL_EXTENT,
other=0.0,
)
row_offsets = tl.arange(0, BLOCK_M)
for group_offset in tl.static_range(GROUP_M):
row = (row_group * GROUP_M + group_offset) * BLOCK_M + row_offsets
token = row // NUM_Q_HEADS
head = row % NUM_Q_HEADS
relative = tl.load(
qkvr_ptr
+ token[:, None] * stride_x_t
+ R_OFFSET
+ head[:, None] * 16
+ inner[None, :],
mask=row[:, None] < rows,
other=0.0,
)
values = tl.dot(relative, projection, out_dtype=tl.float32).to(
rel_out_ptr.dtype.element_ty
)
values = values.to(tl.float32)
if APPLY_LOG_SCALING:
values *= tl.load(log_scaling_ptr + token, mask=row < rows, other=1.0)[
:, None
]
tl.store(
rel_out_ptr + row[:, None] * REL_EXTENT + col[None, :],
values.to(rel_out_ptr.dtype.element_ty),
mask=(row[:, None] < rows) & (col[None, :] < REL_EXTENT),
)
def use_rel_proj_throughput(rows: int, rel_extent: int) -> bool:
min_rows = 8192 if rel_extent == 512 else 2048
return rows >= min_rows
def qkvr_rel_proj(
qkvr: torch.Tensor,
rel_proj: torch.Tensor,
rel_out: torch.Tensor,
log_scaling: torch.Tensor | None,
*,
num_q_heads: int,
num_kv_heads: int,
head_dim: int,
d_rel: int,
) -> None:
rows = qkvr.shape[0] * num_q_heads
rel_extent = rel_proj.shape[1]
assert d_rel == 16 and rel_proj.shape[0] == 16
r_offset = num_q_heads * head_dim + 2 * num_kv_heads * head_dim
log_scaling_ptr = log_scaling if log_scaling is not None else qkvr
common = dict(
R_OFFSET=r_offset,
NUM_Q_HEADS=num_q_heads,
REL_EXTENT=rel_extent,
APPLY_LOG_SCALING=log_scaling is not None,
)
if use_rel_proj_throughput(rows, rel_extent):
grid = (
triton.cdiv(rows, THROUGHPUT_BLOCK_M * THROUGHPUT_GROUP_M),
triton.cdiv(rel_extent, THROUGHPUT_BLOCK_N),
)
_rel_proj_throughput_kernel[grid](
qkvr,
rel_proj,
rel_out,
log_scaling_ptr,
rows,
qkvr.stride(0),
BLOCK_M=THROUGHPUT_BLOCK_M,
BLOCK_N=THROUGHPUT_BLOCK_N,
GROUP_M=THROUGHPUT_GROUP_M,
num_warps=THROUGHPUT_NUM_WARPS,
**common,
)
return
grid = (
triton.cdiv(rows, LOW_BLOCK_M),
triton.cdiv(rel_extent, LOW_BLOCK_N),
)
_rel_proj_low_latency_kernel[grid](
qkvr,
rel_proj,
rel_out,
log_scaling_ptr,
rows,
qkvr.stride(0),
BLOCK_M=LOW_BLOCK_M,
BLOCK_N=LOW_BLOCK_N,
num_warps=LOW_NUM_WARPS,
**common,
)
@triton.jit(do_not_specialize=["tokens", "stride_block_table_req", "max_blocks"])
def _qkvr_qkv_kernel(
qkvr_ptr,
q_norm_weight_ptr,
q_out_ptr,
rel_proj_ptr,
rel_out_ptr,
k_weight_ptr,
v_weight_ptr,
k_norm_weight_ptr,
conv_cache_ptr,
key_cache_ptr,
value_cache_ptr,
positions_ptr,
seq_idx_ptr,
conv_slot_mapping_ptr,
conv_block_table_ptr,
query_start_ptr,
attention_slot_mapping_ptr,
log_scaling_ptr,
tokens,
eps,
stride_x_t,
stride_cc_block,
stride_cc_head,
stride_cc_token,
stride_cc_dim,
stride_kc_block,
stride_kc_token,
stride_kc_head,
stride_vc_block,
stride_vc_token,
stride_vc_head,
stride_block_table_req,
max_blocks,
conv_block_size,
attention_page_size,
Q_WIDTH: tl.constexpr,
KV_WIDTH: tl.constexpr,
NUM_Q_HEADS: tl.constexpr,
NUM_KV_HEADS: tl.constexpr,
HEAD_DIM: tl.constexpr,
WINDOW_SIZE: tl.constexpr,
OFF_K: tl.constexpr,
OFF_V: tl.constexpr,
APPLY_LOG_SCALING: tl.constexpr,
D_REL: tl.constexpr,
REL_EXTENT: tl.constexpr,
REL_EXTENT_PADDED: tl.constexpr,
):
block = tl.program_id(0)
num_q_rows = tokens * NUM_Q_HEADS
dims = tl.arange(0, HEAD_DIM)
if block < num_q_rows:
row = block
token = row // NUM_Q_HEADS
head = row % NUM_Q_HEADS
values = tl.load(
qkvr_ptr + token * stride_x_t + head * HEAD_DIM + dims,
).to(tl.float32)
weight = tl.load(q_norm_weight_ptr + dims).to(tl.float32)
rstd = tl.rsqrt(tl.sum(values * values, axis=0) / HEAD_DIM + eps)
normalized = values * rstd * weight
if APPLY_LOG_SCALING:
normalized = normalized.to(q_out_ptr.dtype.element_ty).to(tl.float32)
normalized *= tl.load(log_scaling_ptr + token)
tl.store(
q_out_ptr + row * HEAD_DIM + dims,
normalized.to(q_out_ptr.dtype.element_ty),
)
rel_cols = tl.arange(0, REL_EXTENT_PADDED)
rel_mask = rel_cols < REL_EXTENT
projected = tl.zeros([REL_EXTENT_PADDED], dtype=tl.float32)
rel_offset = Q_WIDTH + 2 * KV_WIDTH + head * D_REL
for rel_dim in tl.static_range(D_REL):
rel_value = tl.load(
qkvr_ptr + token * stride_x_t + rel_offset + rel_dim
).to(tl.float32)
proj = tl.load(
rel_proj_ptr + rel_dim * REL_EXTENT + rel_cols,
mask=rel_mask,
other=0.0,
).to(tl.float32)
projected += rel_value * proj
projected = projected.to(rel_out_ptr.dtype.element_ty).to(tl.float32)
if APPLY_LOG_SCALING:
projected *= tl.load(log_scaling_ptr + token)
tl.store(
rel_out_ptr + row * REL_EXTENT + rel_cols,
projected.to(rel_out_ptr.dtype.element_ty),
mask=rel_mask,
)
else:
row = block - num_q_rows
if row < tokens * NUM_KV_HEADS:
token = row // NUM_KV_HEADS
head = row % NUM_KV_HEADS
position = tl.load(positions_ptr + token)
request = tl.load(seq_idx_ptr + token)
conv_slot = tl.load(conv_slot_mapping_ptr + token)
query_start = tl.load(query_start_ptr + token)
attention_slot = tl.load(attention_slot_mapping_ptr + token)
valid = conv_slot >= 0
k_col = Q_WIDTH + head * HEAD_DIM
v_col = Q_WIDTH + KV_WIDTH + head * HEAD_DIM
k_value = tl.load(qkvr_ptr + token * stride_x_t + k_col + dims)
v_value = tl.load(qkvr_ptr + token * stride_x_t + v_col + dims)
safe_slot = tl.maximum(conv_slot, 0)
cache_base = (
conv_cache_ptr
+ (safe_slot // conv_block_size) * stride_cc_block
+ head * stride_cc_head
+ (safe_slot % conv_block_size) * stride_cc_token
)
tl.store(
cache_base + (OFF_K + dims) * stride_cc_dim,
k_value,
mask=valid,
)
tl.store(
cache_base + (OFF_V + dims) * stride_cc_dim,
v_value,
mask=valid,
)
acc_k = tl.zeros([HEAD_DIM], dtype=tl.float32)
acc_v = tl.zeros([HEAD_DIM], dtype=tl.float32)
for tap in tl.static_range(WINDOW_SIZE):
source_position = position - (WINDOW_SIZE - 1) + tap
source_row = token - (WINDOW_SIZE - 1) + tap
in_window = valid & (source_position >= 0)
intra = in_window & (source_row >= query_start)
cached = in_window & (source_row < query_start)
safe_row = tl.maximum(source_row, 0)
source_k = tl.load(
qkvr_ptr + safe_row * stride_x_t + k_col + dims,
mask=intra,
other=0.0,
).to(tl.float32)
source_v = tl.load(
qkvr_ptr + safe_row * stride_x_t + v_col + dims,
mask=intra,
other=0.0,
).to(tl.float32)
safe_position = tl.maximum(source_position, 0)
logical_block = tl.minimum(
safe_position // conv_block_size, max_blocks - 1
)
physical_block = tl.load(
conv_block_table_ptr
+ request * stride_block_table_req
+ logical_block,
mask=cached,
other=0,
).to(tl.int64)
tap_base = (
conv_cache_ptr
+ physical_block * stride_cc_block
+ head * stride_cc_head
+ (safe_position % conv_block_size) * stride_cc_token
)
source_k += tl.load(
tap_base + (OFF_K + dims) * stride_cc_dim,
mask=cached,
other=0.0,
).to(tl.float32)
source_v += tl.load(
tap_base + (OFF_V + dims) * stride_cc_dim,
mask=cached,
other=0.0,
).to(tl.float32)
k_weight = tl.load(
k_weight_ptr + (head * HEAD_DIM + dims) * WINDOW_SIZE + tap
).to(tl.float32)
v_weight = tl.load(
v_weight_ptr + (head * HEAD_DIM + dims) * WINDOW_SIZE + tap
).to(tl.float32)
acc_k += source_k * k_weight
acc_v += source_v * v_weight
k_rounded = (acc_k + k_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
v_rounded = (acc_v + v_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
k_float = k_rounded.to(tl.float32)
k_norm_weight = tl.load(k_norm_weight_ptr + dims).to(tl.float32)
rstd = tl.rsqrt(tl.sum(k_float * k_float, axis=0) / HEAD_DIM + eps)
k_normalized = (k_float * rstd * k_norm_weight).to(
qkvr_ptr.dtype.element_ty
)
safe_attention_slot = tl.maximum(attention_slot, 0)
attention_block = safe_attention_slot // attention_page_size
attention_offset = safe_attention_slot % attention_page_size
tl.store(
key_cache_ptr
+ attention_block * stride_kc_block
+ attention_offset * stride_kc_token
+ head * stride_kc_head
+ dims,
k_normalized,
mask=attention_slot >= 0,
)
tl.store(
value_cache_ptr
+ attention_block * stride_vc_block
+ attention_offset * stride_vc_token
+ head * stride_vc_head
+ dims,
v_rounded,
mask=attention_slot >= 0,
)
@triton.jit(do_not_specialize=["num_rows"])
def _q_kernel(
qkvr_ptr,
q_norm_weight_ptr,
q_out_ptr,
log_scaling_ptr,
num_rows,
stride_x_t,
eps,
NUM_Q_HEADS: tl.constexpr,
HEAD_DIM: tl.constexpr,
BLOCK_ROWS: tl.constexpr,
APPLY_LOG_SCALING: tl.constexpr,
):
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
dims = tl.arange(0, HEAD_DIM)
row_mask = rows < num_rows
tokens = rows // NUM_Q_HEADS
heads = rows % NUM_Q_HEADS
values = tl.load(
qkvr_ptr
+ tokens[:, None] * stride_x_t
+ heads[:, None] * HEAD_DIM
+ dims[None, :],
mask=row_mask[:, None],
other=0.0,
).to(tl.float32)
weight = tl.load(q_norm_weight_ptr + dims).to(tl.float32)
rstd = tl.rsqrt(tl.sum(values * values, axis=1) / HEAD_DIM + eps)
normalized = values * rstd[:, None] * weight[None, :]
if APPLY_LOG_SCALING:
normalized = normalized.to(q_out_ptr.dtype.element_ty).to(tl.float32)
tau = tl.load(log_scaling_ptr + tokens, mask=row_mask, other=1.0)
normalized *= tau[:, None]
tl.store(
q_out_ptr + rows[:, None] * HEAD_DIM + dims[None, :],
normalized.to(q_out_ptr.dtype.element_ty),
mask=row_mask[:, None],
)
@triton.jit(do_not_specialize=["tokens", "stride_block_table_req", "max_blocks"])
def _kv_kernel(
qkvr_ptr,
k_weight_ptr,
v_weight_ptr,
k_norm_weight_ptr,
conv_cache_ptr,
key_cache_ptr,
value_cache_ptr,
positions_ptr,
seq_idx_ptr,
conv_slot_mapping_ptr,
conv_block_table_ptr,
query_start_ptr,
attention_slot_mapping_ptr,
tokens,
eps,
stride_x_t,
stride_cc_block,
stride_cc_head,
stride_cc_token,
stride_cc_dim,
stride_kc_block,
stride_kc_token,
stride_kc_head,
stride_vc_block,
stride_vc_token,
stride_vc_head,
stride_block_table_req,
max_blocks,
conv_block_size,
attention_page_size,
Q_WIDTH: tl.constexpr,
KV_WIDTH: tl.constexpr,
NUM_KV_HEADS: tl.constexpr,
HEAD_DIM: tl.constexpr,
WINDOW_SIZE: tl.constexpr,
OFF_K: tl.constexpr,
OFF_V: tl.constexpr,
BLOCK_ROWS: tl.constexpr,
):
token = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
dims = tl.arange(0, HEAD_DIM)
row_mask = token < tokens
head_id = tl.program_id(1)
head = tl.full([BLOCK_ROWS], head_id, tl.int64)
position = tl.load(positions_ptr + token, mask=row_mask, other=0)
request = tl.load(seq_idx_ptr + token, mask=row_mask, other=0)
conv_slot = tl.load(conv_slot_mapping_ptr + token, mask=row_mask, other=-1)
query_start = tl.load(query_start_ptr + token, mask=row_mask, other=0)
attention_slot = tl.load(
attention_slot_mapping_ptr + token, mask=row_mask, other=-1
)
valid = row_mask & (conv_slot >= 0)
k_col = Q_WIDTH + head * HEAD_DIM
v_col = Q_WIDTH + KV_WIDTH + head * HEAD_DIM
k_value = tl.load(
qkvr_ptr + token[:, None] * stride_x_t + k_col[:, None] + dims[None, :],
mask=row_mask[:, None],
other=0.0,
)
v_value = tl.load(
qkvr_ptr + token[:, None] * stride_x_t + v_col[:, None] + dims[None, :],
mask=row_mask[:, None],
other=0.0,
)
safe_slot = tl.maximum(conv_slot, 0)
cache_base = (
conv_cache_ptr
+ (safe_slot // conv_block_size) * stride_cc_block
+ head * stride_cc_head
+ (safe_slot % conv_block_size) * stride_cc_token
)
tl.store(
cache_base[:, None] + (OFF_K + dims[None, :]) * stride_cc_dim,
k_value,
mask=valid[:, None],
)
tl.store(
cache_base[:, None] + (OFF_V + dims[None, :]) * stride_cc_dim,
v_value,
mask=valid[:, None],
)
acc_k = tl.zeros([BLOCK_ROWS, HEAD_DIM], dtype=tl.float32)
acc_v = tl.zeros([BLOCK_ROWS, HEAD_DIM], dtype=tl.float32)
for tap in tl.static_range(WINDOW_SIZE):
source_position = position - (WINDOW_SIZE - 1) + tap
source_row = token - (WINDOW_SIZE - 1) + tap
in_window = valid & (source_position >= 0)
intra = in_window & (source_row >= query_start)
cached = in_window & (source_row < query_start)
safe_row = tl.maximum(source_row, 0)
source_k = tl.load(
qkvr_ptr + safe_row[:, None] * stride_x_t + k_col[:, None] + dims[None, :],
mask=intra[:, None],
other=0.0,
).to(tl.float32)
source_v = tl.load(
qkvr_ptr + safe_row[:, None] * stride_x_t + v_col[:, None] + dims[None, :],
mask=intra[:, None],
other=0.0,
).to(tl.float32)
safe_position = tl.maximum(source_position, 0)
logical_block = tl.minimum(safe_position // conv_block_size, max_blocks - 1)
physical_block = tl.load(
conv_block_table_ptr + request * stride_block_table_req + logical_block,
mask=cached,
other=0,
).to(tl.int64)
tap_base = (
conv_cache_ptr
+ physical_block * stride_cc_block
+ head * stride_cc_head
+ (safe_position % conv_block_size) * stride_cc_token
)
source_k += tl.load(
tap_base[:, None] + (OFF_K + dims[None, :]) * stride_cc_dim,
mask=cached[:, None],
other=0.0,
).to(tl.float32)
source_v += tl.load(
tap_base[:, None] + (OFF_V + dims[None, :]) * stride_cc_dim,
mask=cached[:, None],
other=0.0,
).to(tl.float32)
k_weight = tl.load(
k_weight_ptr + (head_id * HEAD_DIM + dims) * WINDOW_SIZE + tap
).to(tl.float32)
v_weight = tl.load(
v_weight_ptr + (head_id * HEAD_DIM + dims) * WINDOW_SIZE + tap
).to(tl.float32)
acc_k += source_k * k_weight[None, :]
acc_v += source_v * v_weight[None, :]
k_rounded = (acc_k + k_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
v_rounded = (acc_v + v_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
k_float = k_rounded.to(tl.float32)
k_norm_weight = tl.load(k_norm_weight_ptr + dims).to(tl.float32)
rstd = tl.rsqrt(tl.sum(k_float * k_float, axis=1) / HEAD_DIM + eps)
k_normalized = (k_float * rstd[:, None] * k_norm_weight[None, :]).to(
qkvr_ptr.dtype.element_ty
)
safe_attention_slot = tl.maximum(attention_slot, 0)
attention_block = safe_attention_slot // attention_page_size
attention_offset = safe_attention_slot % attention_page_size
attention_mask = row_mask & (attention_slot >= 0)
tl.store(
key_cache_ptr
+ attention_block[:, None] * stride_kc_block
+ attention_offset[:, None] * stride_kc_token
+ head[:, None] * stride_kc_head
+ dims[None, :],
k_normalized,
mask=attention_mask[:, None],
)
tl.store(
value_cache_ptr
+ attention_block[:, None] * stride_vc_block
+ attention_offset[:, None] * stride_vc_token
+ head[:, None] * stride_vc_head
+ dims[None, :],
v_rounded,
mask=attention_mask[:, None],
)
def _run_tiled_q(
qkvr: torch.Tensor,
q_norm_weight: torch.Tensor,
q_out: torch.Tensor,
positions: torch.Tensor,
*,
eps: float,
num_q_heads: int,
head_dim: int,
log_scaling: torch.Tensor | None,
) -> None:
num_rows = qkvr.shape[0] * num_q_heads
_q_kernel[(triton.cdiv(num_rows, Q_BLOCK_ROWS),)](
qkvr,
q_norm_weight,
q_out,
log_scaling if log_scaling is not None else positions,
num_rows,
qkvr.stride(0),
eps,
NUM_Q_HEADS=num_q_heads,
HEAD_DIM=head_dim,
BLOCK_ROWS=Q_BLOCK_ROWS,
APPLY_LOG_SCALING=log_scaling is not None,
num_warps=Q_NUM_WARPS,
)
def _run_tiled_kv(
qkvr: torch.Tensor,
k_weight: torch.Tensor,
v_weight: torch.Tensor,
k_norm_weight: torch.Tensor,
conv_cache: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
positions: torch.Tensor,
seq_idx: torch.Tensor,
conv_slot_mapping: torch.Tensor,
conv_block_table: torch.Tensor,
query_start: torch.Tensor,
attention_slot_mapping: torch.Tensor,
*,
eps: float,
num_q_heads: int,
num_kv_heads: int,
head_dim: int,
off_k: int,
off_v: int,
conv_block_size: int,
) -> None:
tokens = qkvr.shape[0]
_kv_kernel[(triton.cdiv(tokens, KV_BLOCK_ROWS), num_kv_heads)](
qkvr,
k_weight,
v_weight,
k_norm_weight,
conv_cache,
key_cache,
value_cache,
positions,
seq_idx,
conv_slot_mapping,
conv_block_table,
query_start,
attention_slot_mapping,
tokens,
eps,
qkvr.stride(0),
conv_cache.stride(0),
conv_cache.stride(1),
conv_cache.stride(2),
conv_cache.stride(3),
key_cache.stride(0),
key_cache.stride(1),
key_cache.stride(2),
value_cache.stride(0),
value_cache.stride(1),
value_cache.stride(2),
conv_block_table.stride(0),
conv_block_table.shape[1],
conv_block_size,
key_cache.shape[1],
Q_WIDTH=num_q_heads * head_dim,
KV_WIDTH=num_kv_heads * head_dim,
NUM_KV_HEADS=num_kv_heads,
HEAD_DIM=head_dim,
WINDOW_SIZE=k_weight.shape[1],
OFF_K=off_k,
OFF_V=off_v,
BLOCK_ROWS=KV_BLOCK_ROWS,
num_warps=KV_NUM_WARPS,
)
def _run_fused_small(
qkvr: torch.Tensor,
q_norm_weight: torch.Tensor,
q_out: torch.Tensor,
rel_proj: torch.Tensor,
rel_out: torch.Tensor,
k_weight: torch.Tensor,
v_weight: torch.Tensor,
k_norm_weight: torch.Tensor,
conv_cache: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
positions: torch.Tensor,
seq_idx: torch.Tensor,
conv_slot_mapping: torch.Tensor,
conv_block_table: torch.Tensor,
query_start: torch.Tensor,
attention_slot_mapping: torch.Tensor,
*,
eps: float,
num_q_heads: int,
num_kv_heads: int,
head_dim: int,
off_k: int,
off_v: int,
conv_block_size: int,
log_scaling: torch.Tensor | None,
) -> None:
tokens = qkvr.shape[0]
num_q_rows = tokens * num_q_heads
grid = (num_q_rows + tokens * num_kv_heads,)
_qkvr_qkv_kernel[grid](
qkvr,
q_norm_weight,
q_out,
rel_proj,
rel_out,
k_weight,
v_weight,
k_norm_weight,
conv_cache,
key_cache,
value_cache,
positions,
seq_idx,
conv_slot_mapping,
conv_block_table,
query_start,
attention_slot_mapping,
log_scaling if log_scaling is not None else positions,
tokens,
eps,
qkvr.stride(0),
conv_cache.stride(0),
conv_cache.stride(1),
conv_cache.stride(2),
conv_cache.stride(3),
key_cache.stride(0),
key_cache.stride(1),
key_cache.stride(2),
value_cache.stride(0),
value_cache.stride(1),
value_cache.stride(2),
conv_block_table.stride(0),
conv_block_table.shape[1],
conv_block_size,
key_cache.shape[1],
Q_WIDTH=num_q_heads * head_dim,
KV_WIDTH=num_kv_heads * head_dim,
NUM_Q_HEADS=num_q_heads,
NUM_KV_HEADS=num_kv_heads,
HEAD_DIM=head_dim,
WINDOW_SIZE=k_weight.shape[1],
OFF_K=off_k,
OFF_V=off_v,
APPLY_LOG_SCALING=log_scaling is not None,
D_REL=16,
REL_EXTENT=rel_proj.shape[1],
REL_EXTENT_PADDED=triton.next_power_of_2(rel_proj.shape[1]),
num_warps=SMALL_NUM_WARPS,
)
def fused_qkvr_prep(
qkvr: torch.Tensor,
k_weight: torch.Tensor,
v_weight: torch.Tensor,
q_norm_weight: torch.Tensor,
k_norm_weight: torch.Tensor,
rel_proj: torch.Tensor,
eps: float,
num_q_heads: int,
num_kv_heads: int,
head_dim: int,
d_rel: int,
conv_cache: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
positions: torch.Tensor,
conv_block_table: torch.Tensor,
seq_idx: torch.Tensor,
conv_slot_mapping: torch.Tensor,
query_start: torch.Tensor,
attention_slot_mapping: torch.Tensor,
off_k: int,
off_v: int,
conv_block_size: int,
log_scaling: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert d_rel == 16 and rel_proj.shape[0] == 16
assert head_dim == 128
assert qkvr.is_contiguous()
assert k_weight.stride() == (k_weight.shape[1], 1)
assert v_weight.stride() == (v_weight.shape[1], 1)
assert rel_proj.stride() == (rel_proj.shape[1], 1)
assert conv_cache.stride(3) == 1
assert key_cache.stride(3) == 1 and value_cache.stride(3) == 1
tokens = qkvr.shape[0]
q_out = torch.empty(
(tokens, num_q_heads * head_dim), dtype=qkvr.dtype, device=qkvr.device
)
rel_out = torch.empty(
(tokens, num_q_heads, rel_proj.shape[1]),
dtype=qkvr.dtype,
device=qkvr.device,
)
if tokens == 0:
return q_out, rel_out
if tokens < SMALL_TOKEN_THRESHOLD:
_run_fused_small(
qkvr,
q_norm_weight,
q_out,
rel_proj,
rel_out,
k_weight,
v_weight,
k_norm_weight,
conv_cache,
key_cache,
value_cache,
positions,
seq_idx,
conv_slot_mapping,
conv_block_table,
query_start,
attention_slot_mapping,
eps=eps,
num_q_heads=num_q_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
off_k=off_k,
off_v=off_v,
conv_block_size=conv_block_size,
log_scaling=log_scaling,
)
return q_out, rel_out
kv_stream = aux_stream()
assert kv_stream is not None
current_stream = torch.cuda.current_stream()
kv_stream.wait_stream(current_stream)
with torch.cuda.stream(kv_stream):
_run_tiled_kv(
qkvr,
k_weight,
v_weight,
k_norm_weight,
conv_cache,
key_cache,
value_cache,
positions,
seq_idx,
conv_slot_mapping,
conv_block_table,
query_start,
attention_slot_mapping,
eps=eps,
num_q_heads=num_q_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
off_k=off_k,
off_v=off_v,
conv_block_size=conv_block_size,
)
_run_tiled_q(
qkvr,
q_norm_weight,
q_out,
positions,
eps=eps,
num_q_heads=num_q_heads,
head_dim=head_dim,
log_scaling=log_scaling,
)
qkvr_rel_proj(
qkvr,
rel_proj,
rel_out,
log_scaling,
num_q_heads=num_q_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
d_rel=d_rel,
)
current_stream.wait_stream(kv_stream)
return q_out, rel_out
@@ -0,0 +1,403 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Split-KV decode for Inkling relative attention on ROCm.
Adapted from LightSeek TokenSpeed's portable Triton relative-MHA decode.
"""
from __future__ import annotations
import os
import torch
from vllm.triton_utils import tl, triton
_MIN_BLOCK_KV = tl.constexpr(32)
def decode_split_count(max_kv_len: int, window_left: int) -> int:
"""Return the number of parallel KV partitions for decode."""
if window_left >= 0:
effective_len = max(1, min(max_kv_len, window_left + 1))
return min(4, max(1, triton.cdiv(effective_len, 128)))
return min(32, max(1, triton.cdiv(max(1, max_kv_len), 2048)))
def use_split_kv_decode(
*,
max_query_len: int,
max_kv_len: int,
page_size: int,
window_left: int,
) -> bool:
"""Select split-KV only where it outperforms the single-pass kernel."""
if os.getenv("INKLING_SPLIT_KV", "1") != "1":
return False
if max_query_len != 1:
return False
if page_size >= 64:
return True
if window_left >= 0:
return page_size >= 32
return max_kv_len >= 8192
@triton.jit
def _split_kv_stage1(
q_ptr,
rel_ptr,
k_ptr,
v_ptr,
block_table_ptr,
cache_seqlens,
mid_out_ptr,
mid_lse_ptr,
softmax_scale,
stride_q_t,
stride_q_h,
stride_k_b,
stride_k_p,
stride_k_h,
stride_k_d,
stride_v_b,
stride_v_p,
stride_v_h,
stride_v_d,
stride_r_t,
stride_r_h,
stride_r_e,
stride_mo_t,
stride_mo_h,
stride_mo_s,
stride_ml_t,
stride_ml_h,
stride_ml_s,
stride_bt_b: tl.constexpr,
page_size: tl.constexpr,
window_left: tl.constexpr,
gqa_group_size: tl.constexpr,
num_q_heads: tl.constexpr,
head_dim: tl.constexpr,
rel_extent: tl.constexpr,
max_kv_splits: tl.constexpr,
BLOCK_D: tl.constexpr,
BLOCK_H: tl.constexpr,
BLOCK_N: tl.constexpr,
):
pid_t = tl.program_id(0)
pid_h = tl.program_id(1)
pid_s = tl.program_id(2)
head_blocks_per_kv = tl.cdiv(gqa_group_size, BLOCK_H)
kv_head = pid_h // head_blocks_per_kv
valid_block_h: tl.constexpr = min(BLOCK_H, gqa_group_size)
off_h = pid_h * valid_block_h + tl.arange(0, BLOCK_H)
h_valid = (off_h < (pid_h + 1) * valid_block_h) & (off_h < num_q_heads)
off_d = tl.arange(0, BLOCK_D)
d_valid = off_d < head_dim
cache_len = tl.load(cache_seqlens + pid_t)
effective_len = (
tl.minimum(cache_len, window_left + 1) if window_left >= 0 else cache_len
)
kv_offset = cache_len - effective_len
split_len = (
tl.cdiv(tl.cdiv(effective_len, max_kv_splits), _MIN_BLOCK_KV) * _MIN_BLOCK_KV
)
split_start = split_len * pid_s
split_end = tl.minimum(split_start + split_len, effective_len)
q_offsets = pid_t * stride_q_t + off_h[:, None] * stride_q_h + off_d[None, :]
row_max = tl.full((BLOCK_H,), float("-inf"), dtype=tl.float32)
row_sum = tl.zeros((BLOCK_H,), dtype=tl.float32)
acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32)
if split_end > split_start:
q = tl.load(
q_ptr + q_offsets,
mask=h_valid[:, None] & d_valid[None, :],
other=0.0,
)
for start_n in range(split_start, split_end, BLOCK_N):
off_n = start_n + tl.arange(0, BLOCK_N)
n_valid = off_n < split_end
token_idx = kv_offset + off_n
logical_page = token_idx // page_size
page_offset = token_idx % page_size
physical_page = tl.load(
block_table_ptr + pid_t * stride_bt_b + logical_page,
mask=n_valid,
other=0,
)
k_offsets = (
physical_page[None, :].to(tl.int64) * stride_k_b
+ page_offset[None, :] * stride_k_p
+ kv_head * stride_k_h
+ off_d[:, None] * stride_k_d
)
k = tl.load(
k_ptr + k_offsets,
mask=d_valid[:, None] & n_valid[None, :],
other=0.0,
)
scores = tl.dot(q, k.to(q.dtype)) * softmax_scale
rel_dist = cache_len - 1 - token_idx
rel_valid = (rel_dist >= 0) & (rel_dist < rel_extent)
rel_idx = tl.maximum(0, tl.minimum(rel_dist, rel_extent - 1))
rel_offsets = (
pid_t * stride_r_t
+ off_h[:, None] * stride_r_h
+ rel_idx[None, :] * stride_r_e
)
rel_bias = tl.load(
rel_ptr + rel_offsets,
mask=h_valid[:, None] & rel_valid[None, :] & n_valid[None, :],
other=0.0,
)
scores += rel_bias.to(tl.float32)
scores = tl.where(
h_valid[:, None] & n_valid[None, :],
scores,
float("-inf"),
)
v_offsets = (
physical_page[:, None].to(tl.int64) * stride_v_b
+ page_offset[:, None] * stride_v_p
+ kv_head * stride_v_h
+ off_d[None, :] * stride_v_d
)
v = tl.load(
v_ptr + v_offsets,
mask=n_valid[:, None] & d_valid[None, :],
other=0.0,
)
next_max = tl.maximum(tl.max(scores, axis=1), row_max)
old_scale = tl.exp(row_max - next_max)
probs = tl.exp(scores - next_max[:, None])
acc *= old_scale[:, None]
acc += tl.dot(probs.to(v.dtype), v)
row_sum = row_sum * old_scale + tl.sum(probs, axis=1)
row_max = next_max
mid_out_offsets = (
pid_t * stride_mo_t
+ off_h[:, None] * stride_mo_h
+ pid_s * stride_mo_s
+ off_d[None, :]
)
tl.store(
mid_out_ptr + mid_out_offsets,
acc / row_sum[:, None],
mask=h_valid[:, None] & d_valid[None, :],
)
mid_lse_offsets = (
pid_t * stride_ml_t + off_h * stride_ml_h + pid_s * stride_ml_s
)
tl.store(
mid_lse_ptr + mid_lse_offsets,
row_max + tl.log(row_sum),
mask=h_valid,
)
@triton.jit
def _split_kv_stage2(
mid_out_ptr,
mid_lse_ptr,
out_ptr,
cache_seqlens,
stride_mo_t,
stride_mo_h,
stride_mo_s,
stride_ml_t,
stride_ml_h,
stride_ml_s,
stride_o_t,
stride_o_h,
window_left: tl.constexpr,
head_dim: tl.constexpr,
max_kv_splits: tl.constexpr,
BLOCK_D: tl.constexpr,
):
pid_t = tl.program_id(0)
pid_h = tl.program_id(1)
cache_len = tl.load(cache_seqlens + pid_t)
effective_len = (
tl.minimum(cache_len, window_left + 1) if window_left >= 0 else cache_len
)
split_len = (
tl.cdiv(tl.cdiv(effective_len, max_kv_splits), _MIN_BLOCK_KV) * _MIN_BLOCK_KV
)
off_d = tl.arange(0, BLOCK_D)
d_valid = off_d < head_dim
row_max = -float("inf")
row_sum = 0.0
acc = tl.zeros((BLOCK_D,), dtype=tl.float32)
for split_id in range(max_kv_splits):
split_start = split_len * split_id
split_end = tl.minimum(split_start + split_len, effective_len)
if split_end > split_start:
value = tl.load(
mid_out_ptr
+ pid_t * stride_mo_t
+ pid_h * stride_mo_h
+ split_id * stride_mo_s
+ off_d,
mask=d_valid,
other=0.0,
)
split_lse = tl.load(
mid_lse_ptr
+ pid_t * stride_ml_t
+ pid_h * stride_ml_h
+ split_id * stride_ml_s
)
next_max = tl.maximum(split_lse, row_max)
old_scale = tl.exp(row_max - next_max)
split_scale = tl.exp(split_lse - next_max)
acc = acc * old_scale + value * split_scale
row_sum = row_sum * old_scale + split_scale
row_max = next_max
tl.store(
out_ptr + pid_t * stride_o_t + pid_h * stride_o_h + off_d,
acc / row_sum,
mask=d_valid,
)
@torch.no_grad()
def inkling_rel_attention_split_kv_decode(
q: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
*,
block_table: torch.Tensor,
cache_seqlens: torch.Tensor,
softmax_scale: float,
window_left: int,
rel_extent: int,
rel_logits: torch.Tensor,
max_kv_len: int,
out: torch.Tensor,
) -> torch.Tensor:
"""Run split-KV relative attention for single-token decode."""
num_kv_heads = key_cache.shape[2]
gqa_group_size = q.shape[1] // num_kv_heads
max_kv_splits = decode_split_count(max_kv_len, window_left)
block_d = triton.next_power_of_2(q.shape[2])
block_h = min(8, gqa_group_size)
mid_out = torch.empty(
q.shape[0],
q.shape[1],
max_kv_splits,
q.shape[2],
dtype=torch.float32,
device=q.device,
)
mid_lse = torch.empty(
q.shape[0],
q.shape[1],
max_kv_splits,
dtype=torch.float32,
device=q.device,
)
stage1_grid = (
q.shape[0],
triton.cdiv(q.shape[1], block_h),
max_kv_splits,
)
_split_kv_stage1[stage1_grid](
q,
rel_logits,
key_cache,
value_cache,
block_table,
cache_seqlens,
mid_out,
mid_lse,
softmax_scale,
q.stride(0),
q.stride(1),
key_cache.stride(0),
key_cache.stride(1),
key_cache.stride(2),
key_cache.stride(3),
value_cache.stride(0),
value_cache.stride(1),
value_cache.stride(2),
value_cache.stride(3),
rel_logits.stride(0),
rel_logits.stride(1),
rel_logits.stride(2),
mid_out.stride(0),
mid_out.stride(1),
mid_out.stride(2),
mid_lse.stride(0),
mid_lse.stride(1),
mid_lse.stride(2),
block_table.stride(0),
page_size=key_cache.shape[1],
window_left=window_left,
gqa_group_size=gqa_group_size,
num_q_heads=q.shape[1],
head_dim=q.shape[2],
rel_extent=rel_extent,
max_kv_splits=max_kv_splits,
BLOCK_D=block_d,
BLOCK_H=block_h,
BLOCK_N=key_cache.shape[1],
num_warps=4,
num_stages=1,
)
stage2_grid = (q.shape[0], q.shape[1])
_split_kv_stage2[stage2_grid](
mid_out,
mid_lse,
out,
cache_seqlens,
mid_out.stride(0),
mid_out.stride(1),
mid_out.stride(2),
mid_lse.stride(0),
mid_lse.stride(1),
mid_lse.stride(2),
out.stride(0),
out.stride(1),
window_left=window_left,
head_dim=q.shape[2],
max_kv_splits=max_kv_splits,
BLOCK_D=block_d,
num_warps=4,
num_stages=2,
)
return out
+290
View File
@@ -0,0 +1,290 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling short-convolution kernels backed by a paged sliding-window state cache.
Each layer's 4 conv streams (K, V, attn-output, mlp-output) share one paged KV
cache ``[num_blocks, H, N, D]`` (head-major; see ``sconv_swa_attn.py``). A stream
occupies the contiguous D-sub-range ``[off_s, off_s + ws)`` across all ``H``
heads, so its flat per-token width is ``H * ws`` and that is the conv channel
dim. The cache stores the conv *input* at every absolute position.
``fused_sconv`` is the single-launch path used by the model: per token it writes
the current input to its slot and convolves the ``W`` taps ending at its
absolute position. A tap landing inside the current forward is read from the
immutable input ``x`` (row ``src - pos + pid_t``); only pre-forward taps are read
from the paged cache (window position ``src`` -> physical block via
``block_table[req, src // N]``). The just-written slot is never read back this
step, so there is no write/read hazard within or across programs -- which is
why this needs no decode-vs-prefill split and is valid for prefill / decode /
spec alike.
All kernels address the cache purely by ``(slot, absolute_position)`` and
allocate nothing inside the captured region; their grids depend only on the
token count (``fused_sconv`` on a fixed token/channel tiling), so the same
path replays correctly under eager, breakable PIECEWISE, and FULL cudagraphs
without any data-dependent shape or branch.
"""
from __future__ import annotations
import torch
from vllm.triton_utils import tl, triton
@triton.jit
def _fused_sconv_kernel(
x_ptr, # [T, H*WS] head-major current-token inputs (also residual)
cache_ptr, # [num_blocks, H, N, D] paged (page-strided view)
weight_ptr, # [H*WS, W]
out_ptr, # [T, H*WS]
pos_ptr, # [T] int64 absolute position per token
seq_idx_ptr, # [T] int32 token -> batch request
slot_ptr, # [T] int64 flat slot (block*N + blk_off); < 0 => PAD
block_table_ptr, # [num_reqs, max_blocks] int32 block_table
qstart_ptr, # [T] int32 first x-row of the token's request
T, # num tokens
stride_x_t,
stride_c_blk,
stride_c_h,
stride_c_n,
stride_c_d,
stride_w_d,
stride_w_w,
stride_bt_r,
MAX_BLOCKS,
N, # block_size
W: tl.constexpr,
USE_SILU: tl.constexpr,
USE_RESIDUAL: tl.constexpr,
OFF_S: tl.constexpr,
WS: tl.constexpr,
H: tl.constexpr,
BT: tl.constexpr,
BLOCK_C: tl.constexpr,
):
# Each program owns a [BT tokens, BLOCK_C channels] tile. The flat channel
# index packs all H heads head-major (head = c // WS, in-stream offset =
# c % WS), so one program spans heads -- no per-head launch and no
# next_power_of_2(WS) lane waste.
pid_t = tl.program_id(0)
pid_c = tl.program_id(1)
toff = pid_t * BT + tl.arange(0, BT) # [BT] token rows
coff = pid_c * BLOCK_C + tl.arange(0, BLOCK_C) # [BLOCK_C] flat channels
C = H * WS
t_mask = toff < T
c_mask = coff < C
head = tl.minimum(coff // WS, H - 1) # clamp keeps masked lanes in-buffer
cd = OFF_S + coff % WS # cache D-index of the channel's stream slot
slot = tl.load(slot_ptr + toff, mask=t_mask, other=-1) # [BT]
valid = slot >= 0
pos = tl.load(pos_ptr + toff, mask=t_mask, other=0)
req = tl.load(seq_idx_ptr + toff, mask=t_mask, other=0)
qstart = tl.load(qstart_ptr + toff, mask=t_mask, other=0)
tc_mask = t_mask[:, None] & c_mask[None, :]
# 1) Insert each token's input into its paged slot (skip PAD rows).
xv = tl.load(x_ptr + toff[:, None] * stride_x_t + coff[None, :], mask=tc_mask)
safe_slot = tl.maximum(slot, 0)
dst = (
cache_ptr
+ (safe_slot // N)[:, None] * stride_c_blk
+ head[None, :] * stride_c_h
+ (safe_slot % N)[:, None] * stride_c_n
+ cd[None, :] * stride_c_d
)
tl.store(dst, xv, mask=tc_mask & valid[:, None])
# 2) Convolve the W taps ending at each token's `pos`. Each tap is read from
# `x` if it falls inside this forward (row >= the request's first row), else
# from the paged cache. Exactly one source is unmasked per tap, so we sum both.
acc = tl.zeros([BT, BLOCK_C], dtype=tl.float32)
for iw in tl.static_range(W):
src = pos - (W - 1) + iw # [BT] absolute window position
row = toff - (W - 1) + iw # [BT] x-row of `src` (== src - pos + token)
in_win = valid & (src >= 0)
intra = in_win & (row >= qstart)
cached = in_win & (row < qstart)
# intra-forward tap: read the immutable input x (never the slot we just
# wrote), so there is no write/read hazard.
safe_row = tl.maximum(row, 0)
xt = tl.load(
x_ptr + safe_row[:, None] * stride_x_t + coff[None, :],
mask=c_mask[None, :] & intra[:, None],
other=0.0,
).to(tl.float32)
# pre-forward tap: read from the paged cache via the block table. Clamp
# addressing terms; the load is masked off when out of window.
safe_src = tl.maximum(src, 0)
safe_lblk = tl.minimum(safe_src // N, MAX_BLOCKS - 1)
blk = tl.load(
block_table_ptr + req * stride_bt_r + safe_lblk, mask=cached, other=0
).to(tl.int64)
cbase = (
cache_ptr
+ blk[:, None] * stride_c_blk
+ head[None, :] * stride_c_h
+ (safe_src % N)[:, None] * stride_c_n
+ cd[None, :] * stride_c_d
)
cv = tl.load(cbase, mask=c_mask[None, :] & cached[:, None], other=0.0).to(
tl.float32
)
wv = tl.load(
weight_ptr + coff * stride_w_d + iw * stride_w_w, mask=c_mask, other=0.0
).to(tl.float32)
acc += (xt + cv) * wv[None, :]
if USE_SILU:
acc = acc * tl.sigmoid(acc)
if USE_RESIDUAL:
acc += xv.to(tl.float32)
tl.store(
out_ptr + toff[:, None] * stride_x_t + coff[None, :],
acc.to(out_ptr.dtype.element_ty),
mask=tc_mask,
)
def fused_sconv(
x: torch.Tensor, # [T, H*ws] head-major current-token inputs
weight: torch.Tensor, # [H*ws, W]
cache: torch.Tensor, # [num_blocks, H, N, D] paged
positions: torch.Tensor, # [T] int64 absolute position per token
block_table: torch.Tensor, # [num_reqs, max_blocks] int32
seq_idx: torch.Tensor, # [T] int32 token -> batch request
slot_mapping: torch.Tensor, # [T] int64 flat slot (PAD = -1 => skip)
query_start: torch.Tensor, # [T] int32 first x-row of the token's request
off_s: int,
ws: int,
block_size: int,
activation: str | None = None,
use_residual: bool = True,
) -> torch.Tensor:
"""Single-launch insert + depthwise causal conv1d over the paged cache.
Reads same-forward taps from ``x`` and pre-forward taps from the cache, so
it is race-free in one launch for prefill / decode / spec and cudagraph-safe
under eager / piecewise / full capture.
"""
T = x.shape[0]
out = torch.empty_like(x)
if T == 0:
return out
assert x.is_contiguous()
assert cache.stride(3) == 1, "cache D-dim must be contiguous"
H = cache.shape[1]
W = weight.shape[1]
C = H * ws # flat conv channel dim (all heads, head-major)
# Tile BT tokens x BLOCK_C channels per program: enough work per CTA to
# amortize the per-token addressing, while keeping the grid large for
# prefill. BLOCK_C spans heads so there is no per-head launch. A ~2K-element
# tile at 4 warps measured best on Blackwell; larger tiles spill registers.
BLOCK_C = min(triton.next_power_of_2(C), 256)
BT = 8
grid = (triton.cdiv(T, BT), triton.cdiv(C, BLOCK_C))
_fused_sconv_kernel[grid](
x,
cache,
weight,
out,
positions,
seq_idx,
slot_mapping,
block_table,
query_start,
T,
x.stride(0),
cache.stride(0),
cache.stride(1),
cache.stride(2),
cache.stride(3),
weight.stride(0),
weight.stride(1),
block_table.stride(0),
block_table.shape[1],
block_size,
W=W,
USE_SILU=activation in ("silu", "swish"),
USE_RESIDUAL=use_residual,
OFF_S=off_s,
WS=ws,
H=H,
BT=BT,
BLOCK_C=BLOCK_C,
num_warps=4,
)
return out
@triton.jit
def _seq_metadata_kernel(
qsl_ptr, # [num_reqs + 1] int32 cumulative query start rows
seq_idx_ptr, # [T] int32 out: token -> owning request
query_start_ptr, # [T] int32 out: first x-row of the token's request
num_reqs,
num_actual_tokens,
num_padded_tokens,
n_iters, # ceil(log2(num_reqs)): binary-search depth
BLOCK: tl.constexpr,
):
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
tok = offs.to(tl.int32)
# Largest j in [0, num_reqs) with qsl[j] <= tok.
lo = tl.zeros([BLOCK], tl.int32)
hi = tl.full([BLOCK], num_reqs - 1, tl.int32)
for _ in range(n_iters):
mid = (lo + hi + 1) // 2
below = tl.load(qsl_ptr + mid) <= tok
lo = tl.where(below, mid, lo)
hi = tl.where(below, hi, mid - 1)
actual = offs < num_actual_tokens
padded = offs < num_padded_tokens
query_start = tl.load(qsl_ptr + lo)
tl.store(seq_idx_ptr + offs, tl.where(actual, lo, 0), mask=padded)
tl.store(
query_start_ptr + offs,
tl.where(actual, query_start, 0),
mask=padded,
)
def sconv_seq_metadata(
query_start_loc: torch.Tensor,
num_reqs: int,
num_actual_tokens: int,
seq_idx_out: torch.Tensor,
query_start_out: torch.Tensor,
num_padded_tokens: int | None = None,
) -> None:
"""Fill static per-token seq_idx / query_start buffers in one launch.
Replaces the arange + searchsorted + clamp + gather + 2x copy chain of the
sconv metadata build with a single kernel writing both persistent buffers.
Padded rows are filled with zero and must have ``slot_mapping == -1``.
"""
if num_padded_tokens is None:
num_padded_tokens = num_actual_tokens
if num_padded_tokens < num_actual_tokens:
raise ValueError("num_padded_tokens must cover all actual tokens")
if num_padded_tokens > seq_idx_out.shape[0]:
raise ValueError("seq_idx_out is too small for the padded token count")
if num_padded_tokens > query_start_out.shape[0]:
raise ValueError("query_start_out is too small for the padded token count")
BLOCK = 256
n_iters = (num_reqs - 1).bit_length()
grid = (triton.cdiv(num_padded_tokens, BLOCK),)
_seq_metadata_kernel[grid](
query_start_loc,
seq_idx_out,
query_start_out,
num_reqs,
num_actual_tokens,
num_padded_tokens,
n_iters,
BLOCK=BLOCK,
)
+197
View File
@@ -0,0 +1,197 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SwiGLU kernels for the Inkling MLP layers.
``silu_and_mul_triton``: SiLU-and-mul over the checkpoint's interleaved
fused gate/up layout (dense MLP). ``sink_silu_mul_epilogue``: the sink-expert
variant with the per-expert dequant scale and per-token gamma fused in.
"""
import torch
from vllm.triton_utils import tl, triton
@triton.jit(do_not_specialize=["M"])
def _silu_and_mul_triton_kernel(
gateup_out_ptr,
down_inp_ptr,
M,
N: tl.constexpr,
GRID_SIZE: tl.constexpr,
NUM_STAGES: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_N: tl.constexpr,
EVEN_N: tl.constexpr,
INT64_INDEX: tl.constexpr,
):
start_pid = tl.program_id(0)
if INT64_INDEX:
start_pid = start_pid.to(tl.int64)
M = M.to(tl.int64)
NUM_BLOCKS_N: tl.constexpr = tl.cdiv(N, BLOCK_SIZE_N)
num_blocks_mn = tl.cdiv(M, BLOCK_SIZE_M) * NUM_BLOCKS_N
for pid in tl.range(start_pid, num_blocks_mn, GRID_SIZE, num_stages=NUM_STAGES):
pid_m = pid // NUM_BLOCKS_N
pid_n = pid % NUM_BLOCKS_N
offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
offs_n = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
mask_m = offs_m < M
mask_n = offs_n < N
# Interleaved fused gate/up: [g0, u0, g1, u1, ...].
mask_offs_2n = pid_n * BLOCK_SIZE_N + tl.arange(0, 2 * BLOCK_SIZE_N) // 2
tl.static_assert(BLOCK_SIZE_N % 8 == 0, f"{BLOCK_SIZE_N=}")
mask_2n = mask_offs_2n < N
mask_2n = tl.max_constancy(mask_2n, [16])
offs_2n = pid_n * 2 * BLOCK_SIZE_N + tl.arange(0, 2 * BLOCK_SIZE_N)
offs_m2n = offs_m[:, None] * N * 2 + offs_2n[None, :]
if EVEN_N or pid_n * BLOCK_SIZE_N + BLOCK_SIZE_N <= N:
gateup_out = tl.load(
gateup_out_ptr + offs_m2n, mask=mask_m[:, None], other=0.0
)
else:
mask_m2n = mask_m[:, None] & mask_2n[None, :]
gateup_out = tl.load(gateup_out_ptr + offs_m2n, mask=mask_m2n, other=0.0)
gate_out, up_out = tl.split(
tl.reshape(gateup_out, (BLOCK_SIZE_M, BLOCK_SIZE_N, 2))
)
gate_out = gate_out.to(tl.float32)
up_out = up_out.to(tl.float32)
down_inp = gate_out * tl.sigmoid(gate_out) * up_out
mask_mn = mask_m[:, None] if EVEN_N else mask_m[:, None] & mask_n[None, :]
offs_mn = offs_m[:, None] * N + offs_n[None, :]
tl.store(down_inp_ptr + offs_mn, down_inp, mask=mask_mn)
def silu_and_mul_triton(gateup_output: torch.Tensor) -> torch.Tensor:
"""SiLU-and-mul for the interleaved fused gate/up layout.
Adapted from ``inkling_kernels.activation.silu_and_mul_fwd`` (without MXFP).
"""
assert gateup_output.is_contiguous(), (
f"{gateup_output.shape=} {gateup_output.stride()=}"
)
assert gateup_output.ndim == 2, f"{gateup_output.shape=}"
M = gateup_output.shape[0]
hidden_size = gateup_output.shape[1]
assert hidden_size % 2 == 0, f"{hidden_size=}"
N = hidden_size // 2
down_input = torch.empty(
(M, N), device=gateup_output.device, dtype=gateup_output.dtype
)
if M == 0:
return down_input
BLOCK_SIZE_N = max(8, min(256, triton.next_power_of_2(N)))
if M <= 1:
BLOCK_SIZE_M = 4
elif M <= 256:
BLOCK_SIZE_M = 2
elif M < 4096:
BLOCK_SIZE_M = 4
else:
BLOCK_SIZE_M = 16
BLOCK_SIZE_N = max(8, min(128, triton.next_power_of_2(N)))
max_grid_size = triton.cdiv(M, BLOCK_SIZE_M) * triton.cdiv(N, BLOCK_SIZE_N)
num_sms = torch.cuda.get_device_properties(
gateup_output.device
).multi_processor_count
grid_size = min(num_sms * 4, max_grid_size)
_silu_and_mul_triton_kernel[(grid_size,)](
gateup_out_ptr=gateup_output,
down_inp_ptr=down_input,
M=M,
N=N,
GRID_SIZE=grid_size,
NUM_STAGES=1,
BLOCK_SIZE_M=BLOCK_SIZE_M,
BLOCK_SIZE_N=BLOCK_SIZE_N,
EVEN_N=N % BLOCK_SIZE_N == 0,
INT64_INDEX=gateup_output.nbytes >= 2**31,
num_warps=8,
)
return down_input
@triton.jit(do_not_specialize=["T"])
def _sink_epilogue_kernel(
raw_ptr, # [T, S * 2F] gemm1 output, interleaved g/u pairs per expert block
alpha_ptr, # [S] fp32 per-expert pre-SiLU dequant scale
gamma_ptr, # [T, S] fp32 per-token sink weights (may be strided)
ratio_ptr, # [S] fp32 per-expert post-SiLU scale (gemm2 alpha ratio)
out_ptr, # [T, S * F] output
T,
stride_raw_0,
stride_gamma_0,
F: tl.constexpr,
S: tl.constexpr,
BLOCK_F: tl.constexpr,
):
pid_t = tl.program_id(0).to(tl.int64)
pid_sf = tl.program_id(1)
if pid_t >= T:
return
s = pid_sf // (F // BLOCK_F)
offs_f = (pid_sf % (F // BLOCK_F)) * BLOCK_F + tl.arange(0, BLOCK_F)
base = pid_t * stride_raw_0 + s * 2 * F
gate = tl.load(raw_ptr + base + 2 * offs_f).to(tl.float32)
up = tl.load(raw_ptr + base + 2 * offs_f + 1).to(tl.float32)
alpha = tl.load(alpha_ptr + s)
weight = tl.load(gamma_ptr + pid_t * stride_gamma_0 + s) * tl.load(ratio_ptr + s)
gate *= alpha
up *= alpha
h = gate * tl.sigmoid(gate) * up * weight
tl.store(out_ptr + pid_t * (S * F) + s * F + offs_f, h)
def sink_silu_mul_epilogue(
raw: torch.Tensor, # [T, S * 2F] gemm1 output (interleaved gate/up rows)
alphas: torch.Tensor, # [S] fp32
gammas: torch.Tensor, # [T, S] fp32
ratios: torch.Tensor, # [S] fp32
n_experts: int,
out_dtype: torch.dtype,
) -> torch.Tensor:
"""Fused sink-expert epilogue: silu(g * a_e) * (u * a_e) * (gamma * r_e).
One kernel replaces the per-expert dequant column scale, the SwiGLU, and
the per-token gamma multiply between the two sink GEMMs.
"""
tokens = raw.shape[0]
f = raw.shape[1] // (2 * n_experts)
out = torch.empty((tokens, n_experts * f), device=raw.device, dtype=out_dtype)
if tokens == 0:
return out
# raw may be a column-slice of a padded GEMM output (rows strided).
assert raw.stride(1) == 1 and gammas.stride(1) == 1
# Largest power-of-two divisor of f (f = 768 -> 256), capped at 512.
block_f = min(512, f & (-f))
_sink_epilogue_kernel[(tokens, n_experts * (f // block_f))](
raw,
alphas,
gammas,
ratios,
out,
tokens,
raw.stride(0),
gammas.stride(0),
F=f,
S=n_experts,
BLOCK_F=block_f,
)
return out
+226
View File
@@ -0,0 +1,226 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling short-conv state managed as a sliding-window KV cache.
Each decoder layer owns one ``InklingConvState`` (an ``AttentionLayerBase``) that
emits a single ``SlidingWindowSpec`` for the layer's 4 sconv streams (K, V,
attn-output, mlp-output), packed head-major into one block:
H = num_kv_heads (per-rank), N = block_size = sconv_kernel_size,
D = head_dim(K) + head_dim(V) + hidden/H(attn) + hidden/H(mlp)
``D`` is TP-invariant; per rank we store ``H/TP`` heads of width ``D``. The conv
reads/writes this cache out-of-band via a custom backend; the (smaller) conv page
is padded up to the uniform attention page by ``unify_kv_cache_spec_page_size``.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import ClassVar
import torch
from torch import nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionMetadata,
AttentionMetadataBuilder,
CommonAttentionMetadata,
)
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, SlidingWindowSpec
from .ops.sconv import sconv_seq_metadata
# Stream order within the per-head packed D (== contiguous sub-ranges).
_K, _V, _ATTN, _MLP = 0, 1, 2, 3
@dataclass
class InklingSconvMetadata(AttentionMetadata):
block_table: torch.Tensor # [num_reqs, max_blocks] physical blocks per req
slot_mapping: torch.Tensor # [T] int64 flat slot of each token (-1 => skip)
seq_idx: torch.Tensor # [T] int32 token -> batch request
query_start: torch.Tensor # [T] int32 first x-row of each token's request
class InklingSconvMetadataBuilder(AttentionMetadataBuilder[InklingSconvMetadata]):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
def __init__(
self,
kv_cache_spec: AttentionSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
assert isinstance(kv_cache_spec, SlidingWindowSpec)
# Persistent per-token buffers for CUDA graph capture.
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.seq_idx_buffer = torch.empty(
max_num_tokens, dtype=torch.int32, device=device
)
self.query_start_buffer = torch.empty(
max_num_tokens, dtype=torch.int32, device=device
)
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> InklingSconvMetadata:
num_reqs = common_attn_metadata.num_reqs
num_actual_tokens = int(common_attn_metadata.query_start_loc_cpu[-1])
num_padded_tokens = common_attn_metadata.slot_mapping.shape[0]
assert num_padded_tokens >= num_actual_tokens
# Per-token seq_idx (owning request) and query_start (first x-row of
# that request; the fused kernel uses it to tell same-forward taps,
# read from x, from pre-forward taps, read from cache) in one launch.
sconv_seq_metadata(
common_attn_metadata.query_start_loc,
num_reqs,
num_actual_tokens,
self.seq_idx_buffer,
self.query_start_buffer,
num_padded_tokens,
)
return InklingSconvMetadata(
block_table=common_attn_metadata.block_table_tensor,
slot_mapping=common_attn_metadata.slot_mapping[:num_padded_tokens],
seq_idx=self.seq_idx_buffer[:num_padded_tokens],
query_start=self.query_start_buffer[:num_padded_tokens],
)
class InklingSconvBackend(AttentionBackend):
"""Custom dummy backend for the sconv sliding-window cache management."""
@staticmethod
def get_name() -> str:
return "INKLING_SCONV_SWA"
@classmethod
def indexes_kv_by_block_stride(cls) -> bool:
# num_blocks is the outermost dim (HND, see get_kv_cache_shape), so the
# padded conv page is read through a strided view.
return True
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
# HND, num-blocks-first, head-major: [num_blocks, H, N, D].
return (num_blocks, num_kv_heads, block_size, head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# Identity: physical layout == logical [num_blocks, H, N, D].
if include_num_layers_dimension:
return (0, 1, 2, 3, 4)
return (0, 1, 2, 3)
@staticmethod
def get_impl_cls():
raise NotImplementedError(
"InklingSconvBackend has no attention impl; the conv runs out-of-band."
)
@staticmethod
def get_builder_cls() -> type[InklingSconvMetadataBuilder]:
return InklingSconvMetadataBuilder
class InklingConvState(nn.Module, AttentionLayerBase):
"""Per-decoder-layer owner emitting one sliding-window conv-state spec."""
def __init__(
self,
*,
num_kv_heads: int,
head_dim: int,
hidden_size: int,
kernel_size: int,
prefix: str,
) -> None:
super().__init__()
self.prefix = prefix
# Bound to the manager-allocated paged cache by bind_kv_cache; a
# placeholder until then. Read out-of-band by InklingShortConv.
self.kv_cache = torch.tensor([])
tp_size = get_tensor_model_parallel_world_size()
# Guardrails for the conv-state layout below; only these are exercised.
# tp_size <= num_kv_heads keeps >=1 whole KV head per rank (no
# replication/clamping), so the per-head width stays TP-invariant.
assert tp_size <= num_kv_heads, (
f"sconv SWA cache supports tp_size <= num_kv_heads ({num_kv_heads}), "
f"got {tp_size}"
)
# Per-rank head count; D is TP-invariant (K/V heads and the hidden
# chunk both scale 1/TP together). The attn-/mlp-output sconv streams
# are hidden-sharded: each rank owns its H/tp chunk (the sublayer
# outputs are reduce-scattered / all-gathered around the conv).
self.num_kv_heads = num_kv_heads // tp_size
hidden_per_head = hidden_size // num_kv_heads
# Packed per-head width: K + V + attn-output chunk + mlp-output chunk,
# padded to a power of two so every layer's conv page is the same size
# and an exact multiple of the attention page (the page unifier then
# scales attention block sizes instead of padding).
raw_head_size = 2 * head_dim + 2 * hidden_per_head
self.head_size = 1 << (raw_head_size - 1).bit_length()
self.sliding_window = kernel_size
self.block_size = kernel_size
# Per-head D-sub-range (offset, width) for each stream. Streams share
# the cache; each writes/reads its own width across all H heads.
self.stream_ranges: tuple[tuple[int, int], ...] = (
(0, head_dim), # _K
(head_dim, head_dim), # _V
(2 * head_dim, hidden_per_head), # _ATTN
(2 * head_dim + hidden_per_head, hidden_per_head), # _MLP
)
vllm_config = get_current_vllm_config()
self._dtype = vllm_config.model_config.dtype
assert self._dtype == torch.bfloat16, (
f"sconv SWA cache supports bfloat16 only, got {self._dtype}"
)
# Register in the forward context so the runner enumerates this owner as
# an attention-like layer (get_kv_cache_spec / get_attn_backend).
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def forward(self): ...
@property
def cache_block_size(self) -> int:
"""Return the block size used by cache metadata and physical indexing."""
if self.kv_cache.numel() > 0:
return self.kv_cache.shape[2]
return self.block_size
def get_attn_backend(self) -> type[AttentionBackend]:
return InklingSconvBackend
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
return SlidingWindowSpec(
block_size=self.block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=0, # all 4 streams packed into head_size
dtype=self._dtype,
sliding_window=self.sliding_window,
)
+98
View File
@@ -0,0 +1,98 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling short convolution: depthwise causal conv1d (+ residual) over a paged
sliding-window conv-state cache.
Each decoder layer owns one ``InklingConvState`` (``sconv_swa_attn.py``) holding the
manager-allocated paged cache for the layer's 4 sconv streams (K, V, attn-output,
mlp-output), packed head-major into one block. Each ``InklingShortConv`` is a
stateless weight + kernel launcher that, per forward (positions-addressed, the
same path for prefill / decode / mixed), inserts the current tokens' inputs
into their paged slot and convolves each token against the ``W`` taps ending
at its absolute position, reading pre-forward window positions out of the
paged cache via the block table.
Per-forward metadata (``block_table`` / ``slot_mapping`` / ``seq_idx`` /
``query_start``) is built once by ``InklingSconvMetadataBuilder`` and published under
the owner's prefix in the forward context; the absolute ``positions`` are
threaded in from the model. The insert + conv run in a single ``fused_sconv``
launch (same path for prefill / decode / mixed / spec). All inputs are
fixed-address persistent buffers and the grid is fixed, so the conv replays
correctly under eager, PIECEWISE, and FULL cudagraphs.
"""
from __future__ import annotations
import torch
from torch import nn
from torch.nn.parameter import Parameter
from vllm.distributed import get_tensor_model_parallel_rank
from vllm.forward_context import get_forward_context
from vllm.model_executor.utils import set_weight_attrs
from .ops import fused_sconv
from .sconv_swa_attn import InklingConvState, InklingSconvMetadata
class InklingShortConv(nn.Module):
def __init__(
self, dim: int, kernel_size: int, owner: InklingConvState, stream_idx: int
) -> None:
super().__init__()
self.dim = dim
self.kernel_size = kernel_size
self.owner = owner
self.stream_idx = stream_idx
self.tp_rank = get_tensor_model_parallel_rank()
# Depthwise conv weight; checkpoint stores (dim, 1, W).
self.weight = Parameter(torch.empty(dim, 1, kernel_size), requires_grad=False)
set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None:
if loaded_weight.shape[0] != param.shape[0]:
shard = param.shape[0]
loaded_weight = loaded_weight.narrow(0, self.tp_rank * shard, shard)
param.data.copy_(loaded_weight)
def forward(self, x: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
# x: (num_tokens, dim); positions: (num_tokens,) absolute positions.
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
# Memory-profiling / no metadata: identity (residual).
return x
m = attn_metadata.get(self.owner.prefix)
if m is None:
return x
assert isinstance(m, InklingSconvMetadata)
cache = self.owner.kv_cache
if cache.numel() == 0:
# Cache not yet bound (profiling before KV alloc): identity.
return x
off_s, ws = self.owner.stream_ranges[self.stream_idx]
# The hybrid KV-cache planner can enlarge the logical conv block so
# that its physical page size matches the attention caches (for
# example, W=4 becomes 32 when --block-size=128). Metadata slot
# mappings are built with that enlarged size, so index the bound cache
# with its runtime token dimension rather than the kernel window size.
block_size = self.owner.cache_block_size
x = x.contiguous()
weight = self.weight.squeeze(1) # (dim, W)
return fused_sconv(
x,
weight,
cache,
positions,
m.block_table,
m.seq_idx,
m.slot_mapping,
m.query_start,
off_s,
ws,
block_size,
activation=None,
use_residual=True,
)
+9
View File
@@ -12,10 +12,16 @@ if TYPE_CHECKING or HAS_TRITON:
import triton
import triton.language as tl
import triton.language.extra.libdevice as tldevice
from triton.experimental import gluon
from triton.experimental.gluon import language as gl
from triton.language.core import _aggregate as aggregate # noqa: E501
else:
triton = TritonPlaceholder()
tl = TritonLanguagePlaceholder()
tldevice = TritonLanguagePlaceholder()
gluon = TritonLanguagePlaceholder()
gl = TritonLanguagePlaceholder()
aggregate = TritonLanguagePlaceholder()
from vllm.triton_utils.tensor_descriptor import use_tensor_descriptor
@@ -29,5 +35,8 @@ __all__ = [
"tldevice",
"LOG2E",
"LOGE2",
"gluon",
"gl",
"aggregate",
"use_tensor_descriptor",
]