[ROCm] [Model] Enable TML inkling (#48841)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -0,0 +1,176 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import statistics
|
||||
|
||||
import torch
|
||||
from tabulate import tabulate
|
||||
|
||||
from vllm.models.inkling.nvidia.ops import qkvr_prep
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
def make_inputs(tokens: int, tp_size: int, is_local: bool):
|
||||
torch.manual_seed(0)
|
||||
num_q_heads = 64 // tp_size
|
||||
num_kv_heads = (16 if is_local else 8) // tp_size
|
||||
head_dim = 128
|
||||
d_rel = 16
|
||||
rel_extent = 512 if is_local else 1024
|
||||
page_size = 16
|
||||
num_blocks = (tokens + page_size - 1) // page_size
|
||||
q_width = num_q_heads * head_dim
|
||||
kv_width = num_kv_heads * head_dim
|
||||
r_width = num_q_heads * d_rel
|
||||
device = "cuda"
|
||||
|
||||
qkvr = torch.randn(
|
||||
tokens,
|
||||
q_width + 2 * kv_width + r_width,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
k_weight = torch.randn(kv_width, 4, device=device, dtype=torch.bfloat16)
|
||||
v_weight = torch.randn_like(k_weight)
|
||||
q_norm_weight = torch.randn(head_dim, device=device, dtype=torch.bfloat16)
|
||||
k_norm_weight = torch.randn_like(q_norm_weight)
|
||||
rel_proj = torch.randn(d_rel, rel_extent, device=device, dtype=torch.bfloat16)
|
||||
conv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
num_kv_heads,
|
||||
page_size,
|
||||
2 * head_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
key_cache = torch.empty(
|
||||
num_blocks,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
value_cache = torch.empty_like(key_cache)
|
||||
positions = torch.arange(tokens, device=device, dtype=torch.int64)
|
||||
block_table = torch.arange(num_blocks, device=device, dtype=torch.int32)[None]
|
||||
seq_idx = torch.zeros(tokens, device=device, dtype=torch.int32)
|
||||
slots = torch.arange(tokens, device=device, dtype=torch.int64)
|
||||
query_start = torch.zeros(tokens, device=device, dtype=torch.int32)
|
||||
log_scaling = None
|
||||
if not is_local:
|
||||
effective_n = (positions + 1).to(torch.float32)
|
||||
log_scaling = 1.0 + 0.1 * torch.log(torch.clamp(effective_n / 128000, min=1.0))
|
||||
return (
|
||||
qkvr,
|
||||
k_weight,
|
||||
v_weight,
|
||||
q_norm_weight,
|
||||
k_norm_weight,
|
||||
rel_proj,
|
||||
1e-6,
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
d_rel,
|
||||
conv_cache,
|
||||
key_cache,
|
||||
value_cache,
|
||||
positions,
|
||||
block_table,
|
||||
seq_idx,
|
||||
slots,
|
||||
query_start,
|
||||
slots,
|
||||
0,
|
||||
head_dim,
|
||||
page_size,
|
||||
log_scaling,
|
||||
)
|
||||
|
||||
|
||||
def capture(implementation, inputs):
|
||||
outputs = []
|
||||
|
||||
def run():
|
||||
outputs[:] = implementation.fused_qkvr_prep(*inputs)
|
||||
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
for _ in range(3):
|
||||
run()
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run()
|
||||
torch.accelerator.synchronize()
|
||||
return graph, outputs
|
||||
|
||||
|
||||
def time_graph(graph: torch.cuda.CUDAGraph, warmup: int, repeats: int) -> float:
|
||||
for _ in range(warmup):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(repeats):
|
||||
graph.replay()
|
||||
end.record()
|
||||
end.synchronize()
|
||||
return start.elapsed_time(end) * 1000 / repeats
|
||||
|
||||
|
||||
def benchmark(inputs, args) -> float:
|
||||
graph, _ = capture(qkvr_prep, inputs)
|
||||
return statistics.median(
|
||||
time_graph(graph, args.warmup, args.repeats) for _ in range(args.trials)
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def main(args):
|
||||
rows = []
|
||||
for tp_size in args.tp_sizes:
|
||||
for tokens in args.tokens:
|
||||
for is_local in (True, False):
|
||||
triton_us = benchmark(make_inputs(tokens, tp_size, is_local), args)
|
||||
rows.append(
|
||||
[
|
||||
tp_size,
|
||||
tokens,
|
||||
"local" if is_local else "global",
|
||||
triton_us,
|
||||
]
|
||||
)
|
||||
|
||||
print("Inkling QKVR prep (CUDA graph, median latency)")
|
||||
print(
|
||||
tabulate(
|
||||
rows,
|
||||
headers=[
|
||||
"TP",
|
||||
"tokens",
|
||||
"scope",
|
||||
"Triton (us)",
|
||||
],
|
||||
floatfmt=("d", "d", "", ".2f"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = FlexibleArgumentParser()
|
||||
parser.add_argument(
|
||||
"--tokens",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[1 << power for power in range(15)],
|
||||
)
|
||||
parser.add_argument("--tp-sizes", type=int, nargs="+", default=[4, 8])
|
||||
parser.add_argument("--warmup", type=int, default=20)
|
||||
parser.add_argument("--repeats", type=int, default=200)
|
||||
parser.add_argument("--trials", type=int, default=5)
|
||||
main(parser.parse_args())
|
||||
@@ -0,0 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def require_rocm_cdna4() -> None:
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("requires ROCm")
|
||||
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not on_gfx950():
|
||||
pytest.skip("requires CDNA4")
|
||||
@@ -0,0 +1,59 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm guards for shared Inkling model-definition contracts."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.models.inkling as inkling
|
||||
from vllm.lora.utils import get_supported_lora_modules
|
||||
from vllm.model_executor.models.interfaces import supports_lora
|
||||
from vllm.models.inkling.amd import model as amd_model
|
||||
from vllm.models.inkling.amd.mtp import InklingMTP
|
||||
|
||||
|
||||
def test_backend_neutral_definitions_match_nvidia_reference() -> None:
|
||||
inkling_dir = Path(amd_model.__file__).parents[1]
|
||||
for filename in ("logits_processor.py", "mtp.py"):
|
||||
assert (inkling_dir / "amd" / filename).read_bytes() == (
|
||||
inkling_dir / "nvidia" / filename
|
||||
).read_bytes()
|
||||
|
||||
|
||||
def test_rocm_exports_its_aligned_mtp_implementation() -> None:
|
||||
assert inkling.InklingMTP is InklingMTP
|
||||
|
||||
|
||||
def test_rocm_model_exposes_the_upstream_lora_contract() -> None:
|
||||
model_cls = amd_model._TmlForCausalLMBase
|
||||
|
||||
assert supports_lora(model_cls)
|
||||
assert model_cls.packed_modules_mapping == {
|
||||
"qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"],
|
||||
"w13": ["w1", "w3"],
|
||||
}
|
||||
assert model_cls.embedding_modules == {"lm_head": "output_embeddings"}
|
||||
|
||||
model = torch.nn.Module()
|
||||
model.embedding_modules = model_cls.embedding_modules
|
||||
supported = get_supported_lora_modules(model)
|
||||
assert "embed_tokens" not in supported
|
||||
assert "lm_head" in supported
|
||||
|
||||
|
||||
def test_lightseek_bundled_adapter_weights_remain_opt_in() -> None:
|
||||
mapper = amd_model._TmlForCausalLMBase.hf_to_vllm_mapper
|
||||
weight = torch.empty(1)
|
||||
name, mapped_weight = next(
|
||||
iter(
|
||||
mapper.apply([("language_model.layers.3.attn.wq_du.lora_A.weight", weight)])
|
||||
)
|
||||
)
|
||||
|
||||
assert name == "model.layers.3.attn.qkvr.lora_A.weight"
|
||||
assert mapped_weight.shard_id == 0
|
||||
assert amd_model._is_peft_adapter_weight(name)
|
||||
|
||||
lm_head_name = mapper.apply_list(["language_model.lm_head.lora_B.weight"])
|
||||
assert lm_head_name == ["lm_head.lora_B.weight"]
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for Quark OCP MXFP4 weights loaded into AITER MoE layouts."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.inkling.amd.moe import InklingMoE
|
||||
|
||||
|
||||
def _fake_moe() -> tuple[InklingMoE, SimpleNamespace]:
|
||||
moe = InklingMoE.__new__(InklingMoE)
|
||||
nn.Module.__init__(moe)
|
||||
moe.n_routed_experts = 4
|
||||
routed = SimpleNamespace(
|
||||
moe_config=SimpleNamespace(
|
||||
moe_parallel_config=SimpleNamespace(tp_rank=1, tp_size=2)
|
||||
),
|
||||
# Logical intermediate per rank is 3; AITER pads it to 4.
|
||||
w13_weight=nn.Parameter(torch.zeros(4, 8, 2, dtype=torch.uint8), False),
|
||||
w2_weight=nn.Parameter(torch.zeros(4, 5, 3, dtype=torch.uint8), False),
|
||||
w13_weight_scale=nn.Parameter(torch.ones(4, 8, 2, dtype=torch.uint8), False),
|
||||
w2_weight_scale=nn.Parameter(torch.ones(4, 5, 3, dtype=torch.uint8), False),
|
||||
)
|
||||
moe.experts = SimpleNamespace(routed_experts=routed)
|
||||
moe._local_expert_slots = ( # type: ignore[method-assign]
|
||||
lambda: dict(enumerate(range(4)))
|
||||
)
|
||||
return moe, routed
|
||||
|
||||
|
||||
def test_loads_logical_tp_shards_and_preserves_aiter_padding():
|
||||
moe, routed = _fake_moe()
|
||||
|
||||
w13 = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4, 12, 2)
|
||||
w2 = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4, 5, 4)
|
||||
moe.load_expert_weight("experts.w13_weight", w13)
|
||||
moe.load_expert_weight("experts.w2_weight", w2)
|
||||
|
||||
# TP rank 1 consumes the second six interleaved gate/up rows. AITER keeps
|
||||
# one padding row in each destination half and one padding w2 column.
|
||||
torch.testing.assert_close(routed.w13_weight[:, :3], w13[:, 6:12:2])
|
||||
torch.testing.assert_close(routed.w13_weight[:, 4:7], w13[:, 7:12:2])
|
||||
assert torch.count_nonzero(routed.w13_weight[:, 3]) == 0
|
||||
assert torch.count_nonzero(routed.w13_weight[:, 7]) == 0
|
||||
torch.testing.assert_close(routed.w2_weight[:, :, :2], w2[:, :, 2:4])
|
||||
assert torch.count_nonzero(routed.w2_weight[:, :, 2]) == 0
|
||||
|
||||
|
||||
def test_unflattens_quark_scales_and_preserves_scale_padding():
|
||||
moe, routed = _fake_moe()
|
||||
|
||||
flat_w13_scale = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4 * 12, 2)
|
||||
flat_w2_scale = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4 * 5, 4)
|
||||
moe.load_expert_weight("experts.w13_weight_scale", flat_w13_scale)
|
||||
moe.load_expert_weight("experts.w2_weight_scale", flat_w2_scale)
|
||||
|
||||
w13_scale = flat_w13_scale.view(4, 12, 2)
|
||||
w2_scale = flat_w2_scale.view(4, 5, 4)
|
||||
torch.testing.assert_close(routed.w13_weight_scale[:, :3], w13_scale[:, 6:12:2])
|
||||
torch.testing.assert_close(routed.w13_weight_scale[:, 4:7], w13_scale[:, 7:12:2])
|
||||
assert torch.all(routed.w13_weight_scale[:, 3] == 1)
|
||||
assert torch.all(routed.w13_weight_scale[:, 7] == 1)
|
||||
torch.testing.assert_close(routed.w2_weight_scale[:, :, :2], w2_scale[:, :, 2:4])
|
||||
assert torch.all(routed.w2_weight_scale[:, :, 2] == 1)
|
||||
@@ -0,0 +1,425 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for Inkling's ROCm relative-bias paged attention."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.inkling.amd.ops.fa4_rel_attention import (
|
||||
bucket_max_seqlen_q,
|
||||
inkling_fa4_rel_attention,
|
||||
use_gfx950_gluon_decode,
|
||||
use_gfx950_gluon_extend,
|
||||
)
|
||||
from vllm.models.inkling.amd.ops.rel_attention_decode import (
|
||||
decode_split_count,
|
||||
use_split_kv_decode,
|
||||
)
|
||||
|
||||
HEAD_DIM = 128
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def _reference(
|
||||
q: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
rel_logits: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
q_lens: list[int],
|
||||
kv_lens: list[int],
|
||||
rel_extent: int,
|
||||
window_left: int | None,
|
||||
) -> torch.Tensor:
|
||||
num_heads = q.shape[1]
|
||||
num_kv_heads = key_cache.shape[2]
|
||||
gqa_group = num_heads // num_kv_heads
|
||||
out: list[torch.Tensor] = []
|
||||
q_start = 0
|
||||
|
||||
for req, (q_len, kv_len) in enumerate(zip(q_lens, kv_lens)):
|
||||
page_size = key_cache.shape[1]
|
||||
num_pages = (kv_len + page_size - 1) // page_size
|
||||
page_ids = block_table[req, :num_pages].long()
|
||||
k = key_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
|
||||
v = value_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
|
||||
k = k.float().repeat_interleave(gqa_group, dim=1)
|
||||
v = v.float().repeat_interleave(gqa_group, dim=1)
|
||||
|
||||
q_req = q[q_start : q_start + q_len].float()
|
||||
rel_req = rel_logits[q_start : q_start + q_len].float()
|
||||
scores = torch.einsum("qhd,khd->hqk", q_req, k) / HEAD_DIM
|
||||
|
||||
q_pos = torch.arange(q_len, device=q.device)[:, None] + kv_len - q_len
|
||||
k_pos = torch.arange(kv_len, device=q.device)[None, :]
|
||||
distance = q_pos - k_pos
|
||||
rel_idx = distance.clamp(0, rel_extent - 1)
|
||||
bias = rel_req.permute(1, 0, 2).gather(
|
||||
2, rel_idx[None].expand(num_heads, -1, -1)
|
||||
)
|
||||
in_rel_extent = (distance >= 0) & (distance < rel_extent)
|
||||
scores += torch.where(in_rel_extent[None], bias, 0.0)
|
||||
|
||||
masked = distance < 0
|
||||
if window_left is not None:
|
||||
masked |= distance > window_left
|
||||
scores.masked_fill_(masked[None], float("-inf"))
|
||||
out.append(torch.einsum("hqk,khd->qhd", scores.softmax(-1), v))
|
||||
q_start += q_len
|
||||
|
||||
return torch.cat(out).to(DTYPE)
|
||||
|
||||
|
||||
def test_query_length_bucket():
|
||||
assert bucket_max_seqlen_q(1) == 1
|
||||
assert bucket_max_seqlen_q(17) == 32
|
||||
assert bucket_max_seqlen_q(33) == 64
|
||||
|
||||
|
||||
def test_split_kv_dispatch_thresholds():
|
||||
assert not use_split_kv_decode(
|
||||
max_query_len=4,
|
||||
max_kv_len=65536,
|
||||
page_size=128,
|
||||
window_left=-1,
|
||||
)
|
||||
assert not use_split_kv_decode(
|
||||
max_query_len=1,
|
||||
max_kv_len=4096,
|
||||
page_size=16,
|
||||
window_left=-1,
|
||||
)
|
||||
assert use_split_kv_decode(
|
||||
max_query_len=1,
|
||||
max_kv_len=8192,
|
||||
page_size=16,
|
||||
window_left=-1,
|
||||
)
|
||||
assert use_split_kv_decode(
|
||||
max_query_len=1,
|
||||
max_kv_len=512,
|
||||
page_size=128,
|
||||
window_left=511,
|
||||
)
|
||||
assert decode_split_count(512, 511) == 4
|
||||
assert decode_split_count(65536, -1) == 32
|
||||
|
||||
|
||||
def test_gfx950_gluon_dispatch_is_strict(monkeypatch: pytest.MonkeyPatch):
|
||||
import vllm.models.inkling.amd.ops.fa4_rel_attention as rel_attention
|
||||
|
||||
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
|
||||
assert use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=16, head_dim=128)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=4, page_size=128, head_dim=128)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=256)
|
||||
|
||||
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: False)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
|
||||
|
||||
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
|
||||
assert use_gfx950_gluon_extend(
|
||||
max_query_len=4,
|
||||
max_kv_len=8192,
|
||||
page_size=128,
|
||||
head_dim=128,
|
||||
window_left=-1,
|
||||
)
|
||||
assert not use_gfx950_gluon_extend(
|
||||
max_query_len=4,
|
||||
max_kv_len=8192,
|
||||
page_size=128,
|
||||
head_dim=128,
|
||||
window_left=511,
|
||||
)
|
||||
assert not use_gfx950_gluon_extend(
|
||||
max_query_len=4,
|
||||
max_kv_len=8192,
|
||||
page_size=16,
|
||||
head_dim=128,
|
||||
window_left=-1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("page_size", "window_left", "max_kv_len"),
|
||||
[
|
||||
(16, None, None),
|
||||
(16, 15, None),
|
||||
(128, None, 8192),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_ragged_multi_page_relative_attention(
|
||||
page_size: int,
|
||||
window_left: int | None,
|
||||
max_kv_len: int | None,
|
||||
):
|
||||
"""Covers full/chunked prefill, decode, GQA, and the local window."""
|
||||
torch.manual_seed(19 + int(window_left is not None))
|
||||
device = "cuda"
|
||||
q_lens = [17, 1]
|
||||
kv_lens = [35, 80]
|
||||
num_heads = 8
|
||||
num_kv_heads = 2
|
||||
rel_extent = 16
|
||||
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
|
||||
|
||||
q = torch.randn(sum(q_lens), num_heads, HEAD_DIM, device=device)
|
||||
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
|
||||
key_cache = torch.randn(
|
||||
1 + len(q_lens) * max_pages,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
HEAD_DIM,
|
||||
device=device,
|
||||
)
|
||||
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
|
||||
value_cache = torch.randn_like(key_cache)
|
||||
rel_logits = torch.randn(
|
||||
sum(q_lens), num_heads, rel_extent, device=device, dtype=DTYPE
|
||||
)
|
||||
|
||||
block_table = torch.stack(
|
||||
[
|
||||
torch.arange(1 + req * max_pages, 1 + (req + 1) * max_pages)
|
||||
for req in range(len(q_lens))
|
||||
]
|
||||
).to(device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = torch.tensor(
|
||||
[0, *torch.tensor(q_lens).cumsum(0).tolist()],
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
|
||||
preallocated = torch.empty_like(q)
|
||||
actual = inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=bucket_max_seqlen_q(max(q_lens)),
|
||||
softmax_scale=1.0 / HEAD_DIM,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
max_kv_len=max_kv_len,
|
||||
out=preallocated,
|
||||
)
|
||||
assert actual.data_ptr() == preallocated.data_ptr()
|
||||
|
||||
expected = _reference(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
block_table,
|
||||
q_lens,
|
||||
kv_lens,
|
||||
rel_extent,
|
||||
window_left,
|
||||
)
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("page_size", "kv_lens", "num_kv_heads", "rel_extent", "window_left"),
|
||||
[
|
||||
(16, [257, 513], 1, 64, None),
|
||||
(128, [257, 512], 2, 512, 511),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_split_kv_decode_matches_reference(
|
||||
page_size: int,
|
||||
kv_lens: list[int],
|
||||
num_kv_heads: int,
|
||||
rel_extent: int,
|
||||
window_left: int | None,
|
||||
):
|
||||
"""Covers page-16 split-KV and page-128 gfx950 Gluon decode."""
|
||||
torch.manual_seed(41 + page_size)
|
||||
device = "cuda"
|
||||
batch_size = len(kv_lens)
|
||||
q_lens = [1] * batch_size
|
||||
num_heads = 8
|
||||
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
|
||||
|
||||
q = torch.randn(batch_size, num_heads, HEAD_DIM, device=device)
|
||||
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
|
||||
key_cache = torch.randn(
|
||||
batch_size * max_pages,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
HEAD_DIM,
|
||||
device=device,
|
||||
)
|
||||
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
|
||||
value_cache = torch.randn_like(key_cache)
|
||||
rel_logits = torch.randn(
|
||||
batch_size,
|
||||
num_heads,
|
||||
rel_extent,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
block_table = torch.arange(
|
||||
batch_size * max_pages,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
).view(batch_size, max_pages)
|
||||
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = torch.arange(
|
||||
batch_size + 1,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
max_kv_len = 8192 if page_size == 16 else max(kv_lens)
|
||||
|
||||
actual = inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=1,
|
||||
softmax_scale=1.0 / HEAD_DIM,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
max_kv_len=max_kv_len,
|
||||
)
|
||||
expected = _reference(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
block_table,
|
||||
q_lens,
|
||||
kv_lens,
|
||||
rel_extent,
|
||||
window_left,
|
||||
)
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"q_len",
|
||||
"kv_len",
|
||||
"num_kv_heads",
|
||||
"rel_extent",
|
||||
"window_left",
|
||||
"gluon_enabled",
|
||||
),
|
||||
[
|
||||
(1, 513, 2, 512, 511, True),
|
||||
(1, 513, 2, 512, 511, False),
|
||||
(4, 8192, 1, 1024, None, True),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_gfx950_page128_packed_kv_views_match_reference(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
q_len: int,
|
||||
kv_len: int,
|
||||
num_kv_heads: int,
|
||||
rel_extent: int,
|
||||
window_left: int | None,
|
||||
gluon_enabled: bool,
|
||||
):
|
||||
"""Cover both Gluon and split-KV against vLLM's packed KV allocation."""
|
||||
monkeypatch.setenv(
|
||||
"INKLING_GFX950_GLUON",
|
||||
"1" if gluon_enabled else "0",
|
||||
)
|
||||
torch.manual_seed(71 + q_len)
|
||||
device = "cuda"
|
||||
page_size = 128
|
||||
batch_size = 2
|
||||
num_heads = 8
|
||||
q_lens = [q_len] * batch_size
|
||||
kv_lens = [kv_len] * batch_size
|
||||
pages_per_req = (kv_len + page_size - 1) // page_size
|
||||
|
||||
q = torch.randn(
|
||||
batch_size * q_len,
|
||||
num_heads,
|
||||
HEAD_DIM,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
|
||||
|
||||
# FlashAttentionBackend allocates logical [block, head, page, 2 * dim].
|
||||
# Inkling transposes to [block, page, head, 2 * dim] and splits K/V,
|
||||
# producing non-contiguous views whose page/head strides include both.
|
||||
packed_kv = torch.randn(
|
||||
batch_size * pages_per_req,
|
||||
num_kv_heads,
|
||||
page_size,
|
||||
2 * HEAD_DIM,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
key_cache, value_cache = packed_kv.transpose(1, 2).split(HEAD_DIM, dim=-1)
|
||||
assert not key_cache.is_contiguous()
|
||||
assert key_cache.stride() == value_cache.stride()
|
||||
|
||||
rel_logits = torch.randn(
|
||||
batch_size * q_len,
|
||||
num_heads,
|
||||
rel_extent,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
block_table = torch.arange(
|
||||
batch_size * pages_per_req,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
).view(batch_size, pages_per_req)
|
||||
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = torch.arange(
|
||||
0,
|
||||
batch_size * q_len + 1,
|
||||
q_len,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
|
||||
actual = inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=q_len,
|
||||
softmax_scale=1.0 / HEAD_DIM,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
max_kv_len=kv_len,
|
||||
)
|
||||
expected = _reference(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
block_table,
|
||||
q_lens,
|
||||
kv_lens,
|
||||
rel_extent,
|
||||
window_left,
|
||||
)
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
|
||||
@@ -0,0 +1,116 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm numerical-equivalence tests for the fused MTP depth-layer input kernel.
|
||||
|
||||
``embed_dual_rmsnorm_cat`` must match the unfused module sequence numerically:
|
||||
each rmsnorm computes in fp32 and rounds to bf16 at the same points as the
|
||||
vendored ``rmsnorm`` kernel (including the bf16 round-trip between the
|
||||
chained backbone embed_norm and the depth embed_norm), and the fused row
|
||||
gather matches ``F.embedding``.
|
||||
"""
|
||||
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.inkling.amd.ops.norm import (
|
||||
embed_dual_rmsnorm_cat,
|
||||
embed_rmsnorm,
|
||||
rmsnorm,
|
||||
)
|
||||
|
||||
EPS = 1e-6
|
||||
VOCAB = 4096
|
||||
BF16_ATOL = torch.finfo(torch.bfloat16).eps
|
||||
|
||||
|
||||
def _assert_bf16_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
# Different Triton reduction schedules can move a small number of outputs
|
||||
# by one bf16 epsilon while remaining numerically equivalent.
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=BF16_ATOL)
|
||||
|
||||
|
||||
def _ref(hidden, w_h, w_e, emb, w_pre=None):
|
||||
if w_pre is not None:
|
||||
emb = rmsnorm(emb, w_pre, EPS)
|
||||
return torch.cat([rmsnorm(hidden, w_h, EPS), rmsnorm(emb, w_e, EPS)], dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1536, 6144])
|
||||
@pytest.mark.parametrize("t", [0, 1, 7, 256])
|
||||
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
|
||||
def test_embed_dual_rmsnorm_cat(n: int, t: int, ids_dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(0)
|
||||
dev = "cuda"
|
||||
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
|
||||
w_h = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
w_e = (1 + 0.01 * torch.randn(n, device=dev)).to(torch.bfloat16)
|
||||
w_pre = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
|
||||
hidden = (torch.randn(t, n, device=dev) * 2).to(torch.bfloat16)
|
||||
emb = table[ids.long()]
|
||||
|
||||
# Fused gather + chained backbone pre-norm (the decode draft-step path).
|
||||
out = embed_dual_rmsnorm_cat(
|
||||
hidden,
|
||||
w_h,
|
||||
w_e,
|
||||
EPS,
|
||||
input_ids=ids,
|
||||
embed_table=table,
|
||||
pre_norm_weight=w_pre,
|
||||
)
|
||||
assert out.shape == (t, 2 * n)
|
||||
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb, w_pre))
|
||||
|
||||
# Precomputed embeds, no pre-norm (draft prefill with target-merged MM
|
||||
# embeddings, already backbone-normed).
|
||||
out = embed_dual_rmsnorm_cat(hidden, w_h, w_e, EPS, embeds=emb)
|
||||
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
|
||||
|
||||
# Fused gather, no pre-norm (use_embed_norm=False).
|
||||
out = embed_dual_rmsnorm_cat(
|
||||
hidden, w_h, w_e, EPS, input_ids=ids, embed_table=table
|
||||
)
|
||||
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1536, 6144])
|
||||
@pytest.mark.parametrize("t", [0, 1, 7, 256])
|
||||
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
|
||||
def test_embed_rmsnorm(n: int, t: int, ids_dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(0)
|
||||
dev = "cuda"
|
||||
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
|
||||
w = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
|
||||
ref_emb = table[ids.long()]
|
||||
|
||||
# Gather + embed_norm (base model / MTP prefill embed path).
|
||||
out = cast(torch.Tensor, embed_rmsnorm(ids, table, w, EPS))
|
||||
assert out.shape == (t, n)
|
||||
_assert_bf16_close(out, rmsnorm(ref_emb, w, EPS) if t else ref_emb)
|
||||
|
||||
# Pure gather (use_embed_norm=False / replicated module forward).
|
||||
out = cast(torch.Tensor, embed_rmsnorm(ids, table, None, EPS))
|
||||
assert torch.equal(out, ref_emb)
|
||||
|
||||
# Chained first-layer attn_norm (the target text-path forward): one launch
|
||||
# emits both the residual and layer 0's normed attention input.
|
||||
w_chain = (1 + 0.05 * torch.randn(n, device=dev)).to(torch.bfloat16)
|
||||
res, attn_in = cast(
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
embed_rmsnorm(ids, table, w, EPS, chain_weight=w_chain),
|
||||
)
|
||||
ref_res = rmsnorm(ref_emb, w, EPS) if t else ref_emb
|
||||
_assert_bf16_close(res, ref_res)
|
||||
_assert_bf16_close(attn_in, rmsnorm(ref_res, w_chain, EPS) if t else ref_res)
|
||||
|
||||
# Chained without embed_norm (use_embed_norm=False).
|
||||
res, attn_in = cast(
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
embed_rmsnorm(ids, table, None, EPS, chain_weight=w_chain),
|
||||
)
|
||||
assert torch.equal(res, ref_emb)
|
||||
_assert_bf16_close(attn_in, rmsnorm(ref_emb, w_chain, EPS) if t else ref_emb)
|
||||
@@ -0,0 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.inkling.amd.sconv_swa_attn import InklingConvState
|
||||
|
||||
|
||||
def test_runtime_sconv_block_size_tracks_unified_cache_page():
|
||||
"""The cache planner may enlarge W=4 blocks to match attention pages."""
|
||||
owner = InklingConvState.__new__(InklingConvState)
|
||||
nn.Module.__init__(owner)
|
||||
owner.block_size = 4
|
||||
|
||||
owner.kv_cache = torch.tensor([])
|
||||
assert owner.cache_block_size == 4
|
||||
|
||||
owner.kv_cache = torch.empty(2, 1, 32, 1024)
|
||||
assert owner.cache_block_size == 32
|
||||
@@ -2,12 +2,21 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .nvidia.model import (
|
||||
InklingForCausalLM,
|
||||
InklingForConditionalGeneration,
|
||||
)
|
||||
from .nvidia.mtp import InklingMTP
|
||||
if current_platform.is_rocm():
|
||||
from .amd.model import (
|
||||
InklingForCausalLM,
|
||||
InklingForConditionalGeneration,
|
||||
)
|
||||
from .amd.mtp import InklingMTP as InklingMTP
|
||||
else:
|
||||
from .nvidia.model import (
|
||||
InklingForCausalLM,
|
||||
InklingForConditionalGeneration,
|
||||
)
|
||||
from .nvidia.mtp import InklingMTP as InklingMTP
|
||||
|
||||
__all__ = [
|
||||
"InklingForConditionalGeneration",
|
||||
@@ -18,11 +27,23 @@ __all__ = [
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name == "InklingMTP":
|
||||
from .nvidia import mtp
|
||||
if current_platform.is_rocm():
|
||||
from .amd import mtp as amd_mtp
|
||||
|
||||
return amd_mtp.InklingMTP
|
||||
|
||||
from .nvidia import mtp as nvidia_mtp
|
||||
|
||||
return nvidia_mtp.InklingMTP
|
||||
|
||||
return mtp.InklingMTP
|
||||
if name in __all__:
|
||||
from .nvidia import model
|
||||
if current_platform.is_rocm():
|
||||
from .amd import model as amd_model
|
||||
|
||||
return getattr(amd_model, name)
|
||||
|
||||
from .nvidia import model as nvidia_model
|
||||
|
||||
return getattr(nvidia_model, name)
|
||||
|
||||
return getattr(model, name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,328 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.utils.torch_utils import (
|
||||
canonicalize_singleton_dim_strides,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.attention.backends.flash_attn import (
|
||||
FlashAttentionBackend,
|
||||
FlashAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
from ..configs import InklingModelConfig
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .ops.fa4_rel_attention import (
|
||||
bucket_max_seqlen_q,
|
||||
inkling_fa4_num_splits,
|
||||
inkling_fa4_rel_attention,
|
||||
)
|
||||
from .ops.fa4_warmup import InklingFA4WarmupConfig, register_fa4_warmup
|
||||
from .ops.qkvr_prep import fused_qkvr_prep
|
||||
from .sconv_swa_attn import _K, _V, InklingConvState, InklingSconvMetadata
|
||||
from .short_conv import InklingShortConv
|
||||
|
||||
|
||||
def compute_log_scaling_tau(
|
||||
positions: torch.Tensor, n_floor: int, alpha: float
|
||||
) -> torch.Tensor:
|
||||
effective_n = (positions + 1).to(torch.float32)
|
||||
return 1.0 + alpha * torch.log(torch.clamp(effective_n / float(n_floor), min=1.0))
|
||||
|
||||
|
||||
class RelLogitsProj(nn.Module):
|
||||
"""Project the per-head relative branch ``r`` to per-distance logits."""
|
||||
|
||||
def __init__(self, d_rel: int, rel_extent: int) -> None:
|
||||
super().__init__()
|
||||
self.d_rel = d_rel
|
||||
self.rel_extent = rel_extent
|
||||
self.proj = nn.Parameter(torch.empty(d_rel, rel_extent), requires_grad=False)
|
||||
|
||||
def forward(self, r_out: torch.Tensor) -> torch.Tensor:
|
||||
# r_out: (T, num_heads, d_rel) -> (T, num_heads, rel_extent)
|
||||
return torch.einsum("thd,de->the", r_out, self.proj)
|
||||
|
||||
|
||||
class InklingAttention(nn.Module, AttentionLayerBase):
|
||||
def __init__(
|
||||
self,
|
||||
config: InklingModelConfig,
|
||||
*,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rel_extent: int,
|
||||
local_extent: int,
|
||||
is_local: bool,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
conv_owner: InklingConvState,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.prefix = prefix
|
||||
self.is_local = is_local
|
||||
self.hidden_size = config.hidden_size
|
||||
self.head_dim = head_dim
|
||||
self.d_rel = config.d_rel
|
||||
self.log_scaling_n_floor = config.log_scaling_n_floor
|
||||
self.log_scaling_alpha = config.log_scaling_alpha
|
||||
# q/k are per-head RMS-normed (unit norm), so Inkling scales by 1/head_dim.
|
||||
self.scaling = 1.0 / head_dim
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.num_total_heads = num_heads
|
||||
self.num_total_kv_heads = num_kv_heads
|
||||
assert self.num_total_heads % tp_size == 0
|
||||
self.num_heads = self.num_total_heads // tp_size
|
||||
if self.num_total_kv_heads >= tp_size:
|
||||
assert self.num_total_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % self.num_total_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.num_total_kv_heads // tp_size)
|
||||
# When tp_size > num_kv_heads the K/V projections are padded up to
|
||||
# tp_size heads so each rank gets at least one (GQA replication).
|
||||
kv_total_for_sizing = max(self.num_total_kv_heads, tp_size)
|
||||
|
||||
self.qkvr = MergedColumnParallelLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_sizes=[
|
||||
head_dim * self.num_total_heads,
|
||||
head_dim * kv_total_for_sizing,
|
||||
head_dim * kv_total_for_sizing,
|
||||
self.d_rel * self.num_total_heads,
|
||||
],
|
||||
bias=config.q_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkvr",
|
||||
)
|
||||
self.wo_ud = RowParallelLinear(
|
||||
input_size=head_dim * self.num_total_heads,
|
||||
output_size=config.hidden_size,
|
||||
bias=config.o_bias,
|
||||
quant_config=quant_config,
|
||||
# reduce_results=False: the partial output is all-reduced below
|
||||
# (one-shot custom AR) so the attention-output sconv can run on the
|
||||
# full hidden width fused with the residual add + rmsnorm.
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.wo_ud",
|
||||
)
|
||||
self.rel_extent = local_extent if is_local else rel_extent
|
||||
self.local_extent = local_extent if is_local else None
|
||||
self.rel_logits_proj = RelLogitsProj(self.d_rel, self.rel_extent)
|
||||
self.q_norm = InklingRMSNorm(head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = InklingRMSNorm(head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
# Short convolution on the K/V streams (per-head-width, TP sharded),
|
||||
# applied after the qkvr projection and before q/k norm.
|
||||
kv_conv_dim = self.num_kv_heads * head_dim
|
||||
self.conv_owner = conv_owner
|
||||
self.k_sconv = InklingShortConv(
|
||||
kv_conv_dim, config.sconv_kernel_size, owner=conv_owner, stream_idx=_K
|
||||
)
|
||||
self.v_sconv = InklingShortConv(
|
||||
kv_conv_dim, config.sconv_kernel_size, owner=conv_owner, stream_idx=_V
|
||||
)
|
||||
|
||||
# FA4 left/right window; right=0 keeps it causal. local_extent-1 mirrors
|
||||
# the source (sliding_window_size - 1).
|
||||
self.window_size: tuple[int, int] = (
|
||||
(local_extent - 1, 0) if is_local else (-1, -1)
|
||||
)
|
||||
# Static per-layer-type KV length bound for the split heuristic: local
|
||||
# layers never see more than the sliding window.
|
||||
vllm_config = get_current_vllm_config()
|
||||
self._max_kv_len = (
|
||||
local_extent if is_local else vllm_config.model_config.max_model_len
|
||||
)
|
||||
|
||||
# ---- KV-cache wiring (reuse FlashAttentionBackend for metadata) ----
|
||||
cache_config = vllm_config.cache_config
|
||||
self.kv_cache_dtype = (
|
||||
cache_config.cache_dtype if cache_config is not None else "auto"
|
||||
)
|
||||
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
||||
self.kv_cache_dtype, vllm_config.model_config
|
||||
)
|
||||
self.register_buffer("k_scale", torch.ones((), dtype=torch.float32))
|
||||
self.register_buffer("v_scale", torch.ones((), dtype=torch.float32))
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
self.kv_cache = torch.tensor([]) # replaced by bind_kv_cache
|
||||
|
||||
register_fa4_warmup(
|
||||
InklingFA4WarmupConfig(
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_dim=self.head_dim,
|
||||
rel_extent=self.rel_extent,
|
||||
window_size=self.window_size,
|
||||
is_local=self.is_local,
|
||||
max_kv_len=self._max_kv_len,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
kv_dtype=self.kv_cache_torch_dtype,
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
|
||||
max_num_batched_tokens=(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return FlashAttentionBackend
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
if self.is_local:
|
||||
assert self.local_extent is not None
|
||||
return SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
sliding_window=self.local_extent,
|
||||
)
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
)
|
||||
|
||||
def _split_kv_cache(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
key_cache, value_cache = self.kv_cache.transpose(1, 2).split(
|
||||
self.head_dim, dim=-1
|
||||
)
|
||||
return (
|
||||
canonicalize_singleton_dim_strides(key_cache),
|
||||
canonicalize_singleton_dim_strides(value_cache),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
log_scaling: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = hidden_states.shape[0]
|
||||
qkvr, _ = self.qkvr(hidden_states)
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
attn_output = torch.empty(
|
||||
(num_tokens, self.num_heads, self.head_dim),
|
||||
dtype=qkvr.dtype,
|
||||
device=qkvr.device,
|
||||
)
|
||||
if not isinstance(attn_metadata, dict):
|
||||
attn_output.zero_()
|
||||
else:
|
||||
conv_meta = attn_metadata[self.conv_owner.prefix]
|
||||
md = attn_metadata[self.prefix]
|
||||
assert isinstance(conv_meta, InklingSconvMetadata)
|
||||
fa_md = cast(FlashAttentionMetadata, md)
|
||||
assert self.kv_cache.numel() > 0
|
||||
assert self.conv_owner.kv_cache.numel() > 0
|
||||
# One launch: K/V sconv (conv-cache insert + conv + residual),
|
||||
# Q/K per-head rmsnorm, and the attention KV-cache write. K/V are
|
||||
# consumed via the KV cache; only normed q is materialized.
|
||||
key_cache, value_cache = self._split_kv_cache()
|
||||
off_k, _ = self.conv_owner.stream_ranges[_K]
|
||||
off_v, _ = self.conv_owner.stream_ranges[_V]
|
||||
q, rel_logits = fused_qkvr_prep(
|
||||
qkvr,
|
||||
self.k_sconv.weight.squeeze(1),
|
||||
self.v_sconv.weight.squeeze(1),
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
self.rel_logits_proj.proj,
|
||||
self.q_norm.variance_epsilon,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
self.d_rel,
|
||||
self.conv_owner.kv_cache,
|
||||
key_cache,
|
||||
value_cache,
|
||||
positions,
|
||||
conv_meta.block_table,
|
||||
conv_meta.seq_idx,
|
||||
conv_meta.slot_mapping,
|
||||
conv_meta.query_start,
|
||||
fa_md.slot_mapping,
|
||||
off_k,
|
||||
off_v,
|
||||
self.conv_owner.cache_block_size,
|
||||
log_scaling if not self.is_local else None,
|
||||
)
|
||||
q = q.view(num_tokens, self.num_heads, self.head_dim)
|
||||
self._attention(q, rel_logits, attn_output)
|
||||
|
||||
flat = attn_output.view(num_tokens, -1)
|
||||
output, _ = self.wo_ud(flat)
|
||||
return output
|
||||
|
||||
@eager_break_during_capture
|
||||
def _attention(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
rel_logits: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
assert isinstance(attn_metadata, dict)
|
||||
md = cast(FlashAttentionMetadata, attn_metadata[self.prefix])
|
||||
|
||||
nt = md.num_actual_tokens
|
||||
key_cache, value_cache = self._split_kv_cache()
|
||||
max_seqlen_q = bucket_max_seqlen_q(md.max_query_len)
|
||||
num_splits = inkling_fa4_num_splits(
|
||||
is_local=self.is_local,
|
||||
batch_size=md.seq_lens.shape[0],
|
||||
max_query_len=max_seqlen_q,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
max_kv_len=md.max_seq_len,
|
||||
)
|
||||
inkling_fa4_rel_attention(
|
||||
q[:nt],
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=md.block_table,
|
||||
cache_seqlens=md.seq_lens,
|
||||
cu_seqlens_q=md.query_start_loc,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
softmax_scale=self.scaling,
|
||||
causal=True,
|
||||
window_size=self.window_size,
|
||||
rel_extent=self.rel_extent,
|
||||
rel_logits=rel_logits[:nt],
|
||||
num_splits=num_splits,
|
||||
max_kv_len=md.max_seq_len,
|
||||
out=output[:nt],
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling RMSNorm (no bias, weight-scaled), backed by the vendored Triton kernel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from .ops import rmsnorm
|
||||
|
||||
|
||||
class InklingRMSNorm(nn.Module):
|
||||
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
self.variance_epsilon = eps
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if x.numel() == 0:
|
||||
return x
|
||||
original_shape = x.shape
|
||||
x_2d = x.contiguous().view(-1, self.hidden_size)
|
||||
y = rmsnorm(x_2d, self.weight, self.variance_epsilon)
|
||||
return y.view(original_shape)
|
||||
@@ -0,0 +1,129 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling logits processor (muP + LoRA aware).
|
||||
|
||||
Inkling divides the final logits by a muP width multiplier
|
||||
(``logits_mup_width_multiplier``). This applies it two ways, depending on
|
||||
whether an lm_head LoRA is attached:
|
||||
|
||||
* No LoRA: fold ``1/mup`` into the lm_head GEMM alpha (fp32 epilogue) -- no
|
||||
separate elementwise kernel, no extra rounding, no weight mutation.
|
||||
* LoRA attached: the LoRA manager wraps this layer in
|
||||
``LogitsProcessorWithLoRA``, whose ``forward`` calls
|
||||
``type(base_layer).forward(self=wrapper)`` -- so this ``forward`` runs with
|
||||
``self`` bound to the wrapper. We detect that via ``base_layer`` and take the
|
||||
LoRA path: run the wrapper's ``_get_logits`` (base logits + the lm_head LoRA
|
||||
delta), then divide the full logits by the multiplier so the delta is scaled
|
||||
too. muP thus composes with the LoRA delta, with the dispatch as the only
|
||||
model-side branching.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
|
||||
|
||||
class InklingLogitsProcessor(LogitsProcessor):
|
||||
"""``LogitsProcessor`` that applies Inkling's muP logits width multiplier.
|
||||
|
||||
Args:
|
||||
vocab_size: Padded vocabulary size.
|
||||
org_vocab_size: Unpadded vocabulary size (defaults to ``vocab_size``).
|
||||
scale: Base logits scale (kept ``1.0`` for the served checkpoint).
|
||||
logits_as_input: Whether the input is already logits.
|
||||
soft_cap: Optional logit soft cap (``None`` for the served checkpoint).
|
||||
logits_mup_width_multiplier: muP width divisor for the final logits;
|
||||
``None`` or ``0`` disables it.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int,
|
||||
org_vocab_size: int | None = None,
|
||||
scale: float = 1.0,
|
||||
logits_as_input: bool = False,
|
||||
soft_cap: float | None = None,
|
||||
logits_mup_width_multiplier: float | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
vocab_size=vocab_size,
|
||||
org_vocab_size=org_vocab_size,
|
||||
scale=scale,
|
||||
logits_as_input=logits_as_input,
|
||||
soft_cap=soft_cap,
|
||||
)
|
||||
self.logits_mup_width_multiplier = logits_mup_width_multiplier
|
||||
self._logits_zero: torch.Tensor | None = None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
# ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper,
|
||||
# which calls this forward with ``self`` bound to the wrapper. The
|
||||
# wrapper is not an ``InklingLogitsProcessor`` instance, so dispatch
|
||||
# ``_lora_forward`` explicitly through the base_layer's class (it
|
||||
# provides ``_get_logits``/``logits_as_input``; only ``_lora_forward``
|
||||
# lives on this class).
|
||||
if hasattr(self, "base_layer"):
|
||||
return type(self.base_layer)._lora_forward(
|
||||
self, lm_head, hidden_states, embedding_bias
|
||||
)
|
||||
return self._base_forward(lm_head, hidden_states, embedding_bias)
|
||||
|
||||
def _lora_forward(
|
||||
self,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
# ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits``
|
||||
# returns the base logits plus the lm_head LoRA delta. Apply the muP
|
||||
# divisor on the full logits so the LoRA delta is scaled too.
|
||||
mup_multiplier = self.base_layer.logits_mup_width_multiplier
|
||||
mup = 1.0 / mup_multiplier if mup_multiplier else None
|
||||
if self.logits_as_input:
|
||||
logits = hidden_states
|
||||
else:
|
||||
logits = self._get_logits(hidden_states, lm_head, embedding_bias)
|
||||
# TODO: fuse this multiplication
|
||||
if logits is not None and mup:
|
||||
assert self.base_layer.soft_cap is None
|
||||
assert self.base_layer.scale == 1.0
|
||||
logits = logits * mup
|
||||
return logits
|
||||
|
||||
def _base_forward(
|
||||
self,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
mup = self.logits_mup_width_multiplier
|
||||
if not mup:
|
||||
return super().forward(lm_head, hidden_states, embedding_bias)
|
||||
# Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue):
|
||||
# no separate elementwise kernel, no bf16 rounding of scaled logits, and
|
||||
# no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no
|
||||
# soft cap, unit logits scale.
|
||||
assert self.soft_cap is None
|
||||
assert self.scale == 1.0
|
||||
w = lm_head.weight
|
||||
if self._logits_zero is None:
|
||||
self._logits_zero = w.new_zeros(1)
|
||||
logits = torch.addmm(
|
||||
self._logits_zero,
|
||||
hidden_states,
|
||||
w.t(),
|
||||
beta=0.0,
|
||||
alpha=1.0 / mup,
|
||||
)
|
||||
logits = self._gather_logits(logits)
|
||||
if logits is not None:
|
||||
logits = logits[..., : self.org_vocab_size]
|
||||
return logits
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling dense SwiGLU MLP (also used as the MoE shared expert).
|
||||
|
||||
The checkpoint stores the gate/up projection as a single fused, *interleaved*
|
||||
weight (``[gate0, up0, gate1, up1, ...]``), so we use a plain
|
||||
``ColumnParallelLinear`` whose contiguous row-sharding keeps each gate/up pair
|
||||
together, and an interleaved SwiGLU activation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
|
||||
class InklingDenseMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
*,
|
||||
use_global_scale: bool = False,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.gate_up_proj = ColumnParallelLinear(
|
||||
hidden_size,
|
||||
2 * intermediate_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if use_global_scale:
|
||||
self.global_scale = nn.Parameter(torch.empty(1), requires_grad=False)
|
||||
else:
|
||||
self.global_scale = None
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
from .ops import silu_and_mul_triton
|
||||
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = silu_and_mul_triton(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
if self.global_scale is not None:
|
||||
x.mul_(self.global_scale)
|
||||
# TP-partial output: the layer's reduce-scatter fallback consumes it.
|
||||
return x
|
||||
@@ -0,0 +1,669 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling model implementation for AMD GPUs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.inkling.common.mm_preprocess import (
|
||||
InklingDummyInputsBuilder,
|
||||
InklingMultiModalProcessor,
|
||||
InklingProcessingInfo,
|
||||
inkling_audio_enabled,
|
||||
inkling_vision_enabled,
|
||||
)
|
||||
from vllm.models.inkling.common.towers import InklingAudio, InklingVision
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..configs import InklingMMConfig, InklingModelConfig
|
||||
from .attention import InklingAttention, compute_log_scaling_tau
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .logits_processor import InklingLogitsProcessor
|
||||
from .mlp import InklingDenseMLP
|
||||
from .moe import InklingMoE
|
||||
from .ops.norm import add_rmsnorm, embed_rmsnorm
|
||||
from .sconv_swa_attn import _ATTN, _MLP, InklingConvState
|
||||
from .short_conv import InklingShortConv
|
||||
|
||||
|
||||
def _layer_id(name: str) -> int | None:
|
||||
m = re.search(r"\.layers\.(\d+)\.", name)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _sconv_add_norm(
|
||||
delta: torch.Tensor,
|
||||
hidden: torch.Tensor,
|
||||
sconv: InklingShortConv,
|
||||
norm: InklingRMSNorm | None,
|
||||
positions: torch.Tensor,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""``h = hidden + sconv(TP-sum(delta)); y = rmsnorm(h)``.
|
||||
|
||||
ROCm uses the portable collective path. The Lamport P2P implementation in
|
||||
the NVIDIA model relies on CUDA GDC and CUDA-specific symmetric-memory
|
||||
publication semantics which cannot be linked into a gfx950 HSACO."""
|
||||
norm_w = norm.weight if norm is not None else None
|
||||
eps = norm.variance_epsilon if norm is not None else 0.0
|
||||
|
||||
# RCCL RS -> shard sconv -> AG -> fused add(+rmsnorm).
|
||||
shard = tensor_model_parallel_reduce_scatter(delta, dim=-1)
|
||||
shard = sconv(shard.contiguous(), positions)
|
||||
full = tensor_model_parallel_all_gather(shard, dim=-1)
|
||||
if norm is None:
|
||||
return None, hidden + full
|
||||
return add_rmsnorm(hidden, full, norm_w, eps)
|
||||
|
||||
|
||||
class InklingDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: InklingModelConfig,
|
||||
layer_id: int,
|
||||
is_local: bool,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
force_dense_mlp: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Per-layer owner of the conv state as a paged SWA cache. The 4 sconv
|
||||
# streams (K/V/attn/mlp) are packed head-major into one block and share
|
||||
# it. Built first so the attention layer can wire its K/V sconv to it.
|
||||
self.conv_state = InklingConvState(
|
||||
num_kv_heads=(
|
||||
config.swa_num_key_value_heads
|
||||
if is_local
|
||||
else config.num_key_value_heads
|
||||
),
|
||||
head_dim=config.swa_head_dim if is_local else config.head_dim,
|
||||
hidden_size=config.hidden_size,
|
||||
kernel_size=config.sconv_kernel_size,
|
||||
prefix=f"{prefix}.conv_state",
|
||||
)
|
||||
self.attn_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.attn = InklingAttention(
|
||||
config,
|
||||
num_heads=(
|
||||
config.swa_num_attention_heads
|
||||
if is_local
|
||||
else config.num_attention_heads
|
||||
),
|
||||
num_kv_heads=(
|
||||
config.swa_num_key_value_heads
|
||||
if is_local
|
||||
else config.num_key_value_heads
|
||||
),
|
||||
head_dim=config.swa_head_dim if is_local else config.head_dim,
|
||||
rel_extent=config.rel_extent,
|
||||
local_extent=config.sliding_window_size,
|
||||
is_local=is_local,
|
||||
prefix=f"{prefix}.attn",
|
||||
quant_config=quant_config,
|
||||
conv_owner=self.conv_state,
|
||||
)
|
||||
self.mlp_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
if force_dense_mlp or layer_id < config.dense_mlp_idx:
|
||||
self.mlp: nn.Module = InklingDenseMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.dense_intermediate_size,
|
||||
use_global_scale=config.use_global_scale,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
else:
|
||||
# InklingMoE decides per layer (from the checkpoint exclude list)
|
||||
# whether the routed experts are NVFP4 or bf16; the shared sink
|
||||
# experts are always bf16.
|
||||
self.mlp = InklingMoE(
|
||||
config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
# Short convolution on the attention-output and MLP-output residual
|
||||
# streams, hidden-sharded: the sublayer outputs are reduce-scattered
|
||||
# to [T, H/tp], the sconv runs on the shard, and an all-gather
|
||||
# restores the full residual — all fused with the residual add + next
|
||||
# rmsnorm via the Lamport P2P kernels for decode-sized batches.
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
sconv_dim = config.hidden_size // tp_size
|
||||
self.attn_sconv = InklingShortConv(
|
||||
sconv_dim, config.sconv_kernel_size, owner=self.conv_state, stream_idx=_ATTN
|
||||
)
|
||||
self.mlp_sconv = InklingShortConv(
|
||||
sconv_dim, config.sconv_kernel_size, owner=self.conv_state, stream_idx=_MLP
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
pending: tuple[torch.Tensor | None, InklingShortConv] | None = None,
|
||||
defer_mlp_add: bool = False,
|
||||
attn_in: torch.Tensor | None = None,
|
||||
log_scaling: torch.Tensor | None = None,
|
||||
) -> (
|
||||
torch.Tensor | tuple[torch.Tensor, tuple[torch.Tensor | None, InklingShortConv]]
|
||||
):
|
||||
# The previous sublayer's (pre-reduce, pre-sconv) delta is folded in
|
||||
# fused with its RS/sconv/AG and this layer's pre-attention rmsnorm.
|
||||
# A None delta means the partials sit in the NVLS symm buffer.
|
||||
if pending is None:
|
||||
if attn_in is None:
|
||||
# First layer; on the text path attn_norm comes fused with
|
||||
# the embedding gather (chain_weight in embed_rmsnorm).
|
||||
attn_in = self.attn_norm(hidden_states)
|
||||
else:
|
||||
attn_in, hidden_states = _sconv_add_norm(
|
||||
pending[0], hidden_states, pending[1], self.attn_norm, positions
|
||||
)
|
||||
attn_output = self.attn(positions, attn_in, log_scaling)
|
||||
mlp_in, hidden_states = _sconv_add_norm(
|
||||
attn_output, hidden_states, self.attn_sconv, self.mlp_norm, positions
|
||||
)
|
||||
mlp_output = self.mlp(mlp_in)
|
||||
if defer_mlp_add:
|
||||
# Caller folds mlp_output (pre-reduce, pre-sconv) into the next
|
||||
# fused sconv+add+rmsnorm.
|
||||
return hidden_states, (mlp_output, self.mlp_sconv)
|
||||
# Standalone (MTP) tail: finish the sublayer without a norm.
|
||||
return _sconv_add_norm(
|
||||
mlp_output, hidden_states, self.mlp_sconv, None, positions
|
||||
)[1]
|
||||
|
||||
|
||||
class InklingReplicatedEmbedding(nn.Module):
|
||||
"""Full-vocab embedding table replicated on every TP rank.
|
||||
|
||||
Trades the full table per rank (~2.3 GiB at V=201k / H=6144 bf16, vs a
|
||||
1/tp shard) for no masked lookup and no per-lookup TP all-reduce — one
|
||||
all-reduce per MTP draft step plus one per verify pass — and keeps the
|
||||
full table on-rank for the fused gather+norm kernels (``embed_rmsnorm``,
|
||||
``embed_dual_rmsnorm_cat``). Bit-exact vs vocab-parallel: the all-reduce
|
||||
there only ever summed one real row against exact zeros. The LM head
|
||||
stays vocab-sharded.
|
||||
"""
|
||||
|
||||
def __init__(self, num_embeddings: int, embedding_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(
|
||||
torch.empty(num_embeddings, embedding_dim, dtype=torch.get_default_dtype()),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return embed_rmsnorm(input_ids, self.weight, None, 0.0)
|
||||
|
||||
|
||||
class InklingModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
config: InklingModelConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.embed_tokens = InklingReplicatedEmbedding(
|
||||
config.padded_vocab_size, config.hidden_size
|
||||
)
|
||||
self.embed_norm = (
|
||||
InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
if config.use_embed_norm
|
||||
else None
|
||||
)
|
||||
local_ids = set(config.local_layer_ids)
|
||||
|
||||
def get_layer(prefix: str) -> InklingDecoderLayer:
|
||||
idx = _layer_id(prefix + ".") or int(prefix.split(".")[-1])
|
||||
return InklingDecoderLayer(
|
||||
config, idx, idx in local_ids, quant_config, prefix
|
||||
)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
|
||||
)
|
||||
self.norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], config.hidden_size
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
# Row gather + embed_norm in one launch.
|
||||
norm = self.embed_norm
|
||||
return embed_rmsnorm(
|
||||
input_ids,
|
||||
self.embed_tokens.weight,
|
||||
norm.weight if norm is not None else None,
|
||||
norm.variance_epsilon if norm is not None else 0.0,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
attn_in0: torch.Tensor | None = None
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
# embed_norm was already applied when producing inputs_embeds.
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
# Gather + embed_norm + the first layer's attn_norm, one launch.
|
||||
norm = self.embed_norm
|
||||
hidden_states, attn_in0 = embed_rmsnorm(
|
||||
input_ids,
|
||||
self.embed_tokens.weight,
|
||||
norm.weight if norm is not None else None,
|
||||
self.config.rms_norm_eps,
|
||||
chain_weight=self.layers[self.start_layer].attn_norm.weight,
|
||||
)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
|
||||
log_scaling = None
|
||||
if self.config.log_scaling_n_floor is not None:
|
||||
log_scaling = compute_log_scaling_tau(
|
||||
positions,
|
||||
self.config.log_scaling_n_floor,
|
||||
self.config.log_scaling_alpha,
|
||||
)
|
||||
|
||||
pending: tuple[torch.Tensor | None, InklingShortConv] | None = None
|
||||
for layer in self.layers[self.start_layer : self.end_layer]:
|
||||
hidden_states, pending = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
pending=pending,
|
||||
defer_mlp_add=True,
|
||||
attn_in=attn_in0,
|
||||
log_scaling=log_scaling,
|
||||
)
|
||||
attn_in0 = None
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
if pending is not None:
|
||||
hidden_states = _sconv_add_norm(
|
||||
pending[0], hidden_states, pending[1], None, positions
|
||||
)[1]
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
if pending is not None:
|
||||
# Final RS/sconv/AG + residual add fused with the final rmsnorm.
|
||||
norm_out = _sconv_add_norm(
|
||||
pending[0], hidden_states, pending[1], self.norm, positions
|
||||
)[0]
|
||||
assert norm_out is not None
|
||||
return norm_out
|
||||
return self.norm(hidden_states)
|
||||
|
||||
|
||||
class _TmlForCausalLMBase(nn.Module, SupportsPP, SupportsLoRA):
|
||||
"""Shared text-backbone causal-LM scaffolding for both entry classes."""
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
".w13_dn": ".gate_up_proj",
|
||||
".w2_md": ".down_proj",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".attn.wq_du.": (".attn.qkvr.", 0),
|
||||
".attn.wk_dv.": (".attn.qkvr.", 1),
|
||||
".attn.wv_dv.": (".attn.qkvr.", 2),
|
||||
".attn.wr_du.": (".attn.qkvr.", 3),
|
||||
},
|
||||
orig_to_new_prefix={
|
||||
"model.llm.layers.": "model.layers.",
|
||||
"model.llm.embed_norm": "model.embed_norm",
|
||||
"model.llm.embed": "model.embed_tokens",
|
||||
"model.llm.norm": "model.norm",
|
||||
"model.llm.unembed": "lm_head",
|
||||
"language_model.layers.": "model.layers.",
|
||||
"language_model.lm_head.": "lm_head.",
|
||||
},
|
||||
orig_to_new_suffix={
|
||||
# NVFP4 scale
|
||||
".w13_weight.scale": ".w13_weight_scale",
|
||||
".w13_weight.scale2": ".w13_weight_scale_2",
|
||||
".w2_weight.scale": ".w2_weight_scale",
|
||||
".w2_weight.scale2": ".w2_weight_scale_2",
|
||||
},
|
||||
)
|
||||
# Quark uses this mapping when resolving quantization exclusions for the
|
||||
# four checkpoint attention projections fused into qkvr.
|
||||
packed_modules_mapping = {
|
||||
"qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"],
|
||||
"w13": ["w1", "w3"],
|
||||
}
|
||||
embedding_modules = {
|
||||
"lm_head": "output_embeddings",
|
||||
}
|
||||
|
||||
def _build(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
text_config: InklingModelConfig,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = text_config
|
||||
# ROCm checkpoints use Quark OCP MXFP4. The global Quark config is
|
||||
# passed into each routed MoE and its exclusion list keeps the rest of
|
||||
# Inkling in bf16.
|
||||
# Read by the MRV2 runner to publish per-request short-conv metadata.
|
||||
# Short convolution is intrinsic to Inkling, so this is always set.
|
||||
self.uses_sconv = True
|
||||
self.model = InklingModel(
|
||||
config=text_config,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
)
|
||||
self.lm_head = ParallelLMHead(
|
||||
text_config.padded_vocab_size,
|
||||
text_config.hidden_size,
|
||||
org_num_embeddings=text_config.padded_vocab_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = InklingLogitsProcessor(
|
||||
text_config.padded_vocab_size,
|
||||
org_vocab_size=text_config.vocab_size,
|
||||
soft_cap=text_config.final_logit_softcapping,
|
||||
logits_mup_width_multiplier=text_config.logits_mup_width_multiplier,
|
||||
)
|
||||
self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
return self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
intermediate_tensors,
|
||||
inputs_embeds,
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
return _load_inkling_weights(self, weights, self.config)
|
||||
|
||||
|
||||
class InklingForCausalLM(_TmlForCausalLMBase):
|
||||
"""Text-only entry point (``inkling_model`` checkpoints)."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self._build(vllm_config, vllm_config.model_config.hf_config, prefix)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
InklingMultiModalProcessor,
|
||||
info=InklingProcessingInfo,
|
||||
dummy_inputs=InklingDummyInputsBuilder,
|
||||
)
|
||||
class InklingForConditionalGeneration(_TmlForCausalLMBase, SupportsMultiModal):
|
||||
"""Top-level (multimodal) entry point.
|
||||
|
||||
Builds the vision + audio towers on top of the shared text backbone. Inkling has
|
||||
NO cross-modal fusion (the vision tower emits one token per patch, the audio
|
||||
tower one token per frame), so generation reuses the inherited backbone
|
||||
``forward`` / ``compute_logits`` (the latter already applies muP) and this
|
||||
class only adds multimodal embedding + merge.
|
||||
"""
|
||||
|
||||
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"model.audio.": "audio.",
|
||||
"model.visual.": "visual.vision_encoder.",
|
||||
},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("image"):
|
||||
return "<|content_image|>"
|
||||
if modality.startswith("audio"):
|
||||
return "<|content_audio_input|>"
|
||||
raise ValueError("Only image or audio modality is supported")
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
config: InklingMMConfig = vllm_config.model_config.hf_config
|
||||
|
||||
self.visual = (
|
||||
InklingVision(config.vision_config, prefix=maybe_prefix(prefix, "visual"))
|
||||
if inkling_vision_enabled(config)
|
||||
else None
|
||||
)
|
||||
self.audio = (
|
||||
InklingAudio(config.audio_config, prefix=maybe_prefix(prefix, "audio"))
|
||||
if inkling_audio_enabled(config)
|
||||
else None
|
||||
)
|
||||
|
||||
self._build(vllm_config, config.text_config, prefix)
|
||||
|
||||
# -- multimodal embedding -------------------------------------------
|
||||
|
||||
def _process_image_input(
|
||||
self, pixel_values: Any, num_patches: Any
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
assert self.visual is not None
|
||||
# pixel_values is a list (per item) of [P_i, 2, P, P, 3] tensors,
|
||||
# or a single concatenated tensor. Normalize to a flat batch, run the
|
||||
# tower once, then split back per item.
|
||||
if isinstance(pixel_values, (list, tuple)):
|
||||
if not pixel_values:
|
||||
return ()
|
||||
sizes = [int(p.shape[0]) for p in pixel_values]
|
||||
patches = torch.cat(list(pixel_values), dim=0)
|
||||
else:
|
||||
patches = pixel_values
|
||||
sizes = self._sizes_from(num_patches, patches.shape[0])
|
||||
|
||||
patches = patches.to(device=self.visual.device, dtype=self.visual.dtype)
|
||||
embeds = self.visual(patches) # [total_patches, D]
|
||||
return tuple(embeds.split(sizes))
|
||||
|
||||
def _process_audio_input(
|
||||
self, input_audio_features: Any, num_audio_tokens: Any
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
assert self.audio is not None
|
||||
if isinstance(input_audio_features, (list, tuple)):
|
||||
if not input_audio_features:
|
||||
return ()
|
||||
sizes = [int(d.shape[0]) for d in input_audio_features]
|
||||
dmel = torch.cat(list(input_audio_features), dim=0)
|
||||
else:
|
||||
dmel = input_audio_features
|
||||
sizes = self._sizes_from(num_audio_tokens, dmel.shape[0])
|
||||
|
||||
dmel = dmel.to(device=self.audio.device)
|
||||
embeds = self.audio(dmel) # [total_frames, D]
|
||||
return tuple(embeds.split(sizes))
|
||||
|
||||
@staticmethod
|
||||
def _sizes_from(counts: Any, total: int) -> list[int]:
|
||||
if counts is None:
|
||||
return [total]
|
||||
if isinstance(counts, torch.Tensor):
|
||||
return [int(c) for c in counts.flatten().tolist()]
|
||||
if isinstance(counts, (list, tuple)):
|
||||
flat: list[int] = []
|
||||
for c in counts:
|
||||
flat.append(int(c.item()) if isinstance(c, torch.Tensor) else int(c))
|
||||
return flat
|
||||
return [int(counts)]
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
# Iterate modalities in a stable order so the returned per-item tensors
|
||||
# line up with their appearance order; the positional merge in
|
||||
# embed_input_ids handles actual placement.
|
||||
pixel_values = kwargs.get("pixel_values")
|
||||
num_patches = kwargs.get("num_patches")
|
||||
input_audio_features = kwargs.get("input_audio_features")
|
||||
num_audio_tokens = kwargs.get("num_audio_tokens")
|
||||
|
||||
embeddings: tuple[torch.Tensor, ...] = ()
|
||||
if pixel_values is not None and self.visual is not None:
|
||||
embeddings += self._process_image_input(pixel_values, num_patches)
|
||||
if input_audio_features is not None and self.audio is not None:
|
||||
embeddings += self._process_audio_input(
|
||||
input_audio_features, num_audio_tokens
|
||||
)
|
||||
return embeddings
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Override the base's 1-arg embed_input_ids: the runner calls this 3-arg
|
||||
# signature for multimodal models. Text embeddings come from the shared
|
||||
# backbone (which applies embed_norm); MM embeddings are scattered in.
|
||||
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
|
||||
|
||||
# Placeholder ids use unused vocabulary slots and these positions are
|
||||
# overwritten by MM embeds below.
|
||||
inputs_embeds = self.model.embed_input_ids(input_ids)
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
return inputs_embeds
|
||||
assert is_multimodal is not None
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=inputs_embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def get_language_model(self) -> nn.Module:
|
||||
# This class IS the causal LM (the towers are side branches), so the
|
||||
# language model is self — callers expect a module exposing ``.model``
|
||||
# / ``.lm_head`` (e.g. the MTP/eagle loader shares embeddings via
|
||||
# ``get_language_model().model.embed_tokens``).
|
||||
return self
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Weight loading
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
_MOE_EXPERT_WEIGHT_RE = re.compile(
|
||||
r"^(?P<mlp>.*\.mlp)\.(?P<rest>(?:shared_)?experts\..+)$"
|
||||
)
|
||||
|
||||
|
||||
def _is_peft_adapter_weight(name: str) -> bool:
|
||||
return ".lora_A." in name or ".lora_B." in name
|
||||
|
||||
|
||||
def _load_inkling_weights(
|
||||
module: nn.Module,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
config: InklingModelConfig,
|
||||
) -> set[str]:
|
||||
moe_modules = {
|
||||
name: mod for name, mod in module.named_modules() if isinstance(mod, InklingMoE)
|
||||
}
|
||||
loaded: set[str] = set()
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
local_ids = set(config.local_layer_ids)
|
||||
|
||||
def _iter_loadable_weights() -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, weight in module.hf_to_vllm_mapper.apply(weights):
|
||||
# LightSeek's MXFP4 conversion bundles a quantized copy of the
|
||||
# standalone PEFT adapter in the base safetensors index. These are
|
||||
# not base-model parameters; adapters remain opt-in at serving.
|
||||
if _is_peft_adapter_weight(name):
|
||||
continue
|
||||
shard_id = getattr(weight, "shard_id", None)
|
||||
# Replicate K/V conv-free GQA heads when tp_size > num_kv_heads.
|
||||
if (
|
||||
shard_id in (1, 2)
|
||||
and name.endswith(".attn.qkvr.weight")
|
||||
and weight.shape[0] > 0
|
||||
):
|
||||
lid = _layer_id(name)
|
||||
if lid is not None:
|
||||
is_local = lid in local_ids
|
||||
n_kv = (
|
||||
config.swa_num_key_value_heads
|
||||
if is_local
|
||||
else config.num_key_value_heads
|
||||
)
|
||||
head_dim = config.swa_head_dim if is_local else config.head_dim
|
||||
if tp_size > n_kv and weight.shape[0] == n_kv * head_dim:
|
||||
kv_idx = (tp_rank * n_kv) // tp_size
|
||||
weight = weight.narrow(0, kv_idx * head_dim, head_dim)
|
||||
weight.shard_id = shard_id
|
||||
|
||||
# MoE expert tensors (fused stacked, routed + shared sink): translate
|
||||
# the checkpoint layout to per-expert FusedMoE loads.
|
||||
moe_match = _MOE_EXPERT_WEIGHT_RE.match(name)
|
||||
if moe_match is not None and moe_match.group("mlp") in moe_modules:
|
||||
moe = moe_modules[moe_match.group("mlp")]
|
||||
for rel in moe.load_expert_weight(moe_match.group("rest"), weight):
|
||||
loaded.add(f"{moe_match.group('mlp')}.{rel}")
|
||||
continue
|
||||
|
||||
yield name, weight
|
||||
|
||||
loader = AutoWeightsLoader(module, skip_prefixes=["model.mtp."])
|
||||
loaded |= loader.load_weights(_iter_loadable_weights())
|
||||
|
||||
# Post-load MoE fixups (default input scales, zeroed EP-padding experts).
|
||||
for moe_name, moe in moe_modules.items():
|
||||
for rel in moe.finalize_load():
|
||||
loaded.add(f"{moe_name}.{rel}")
|
||||
return loaded
|
||||
|
||||
|
||||
EntryClass = [InklingForCausalLM, InklingForConditionalGeneration]
|
||||
@@ -0,0 +1,693 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling mixture-of-experts on vLLM's FusedMoE abstraction.
|
||||
|
||||
Overfit to the served checkpoint: sigmoid gate (+ selection bias) top-k over
|
||||
the routed experts, log-sigmoid renormalization over the k routed + S shared
|
||||
"sink" logits, scaled by route_scale * global_scale. The routed top-k goes
|
||||
through vLLM's FusedMoE (which handles TP/EP); the sink experts run in
|
||||
:class:`InklingSinkExperts` -- replicated across EP ranks (every token
|
||||
activates every sink) and always bf16 (the checkpoint excludes every
|
||||
``shared_experts`` from quantization).
|
||||
|
||||
MXFP4 routed experts reuse vLLM's Quark OCP MXFP4 fused-MoE method; excluded
|
||||
(bf16) layers fall back to the unquantized method. The checkpoint's fused
|
||||
stacked tensors (interleaved gate/up rows, ``.scale`` / ``.scale2`` /
|
||||
``.input_amax`` aux tensors) are translated to the standard per-expert loads
|
||||
in :meth:`InklingMoE.load_expert_weight`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_dp_group,
|
||||
get_pcp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, tldevice, triton
|
||||
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
|
||||
from vllm.utils.torch_utils import aux_stream
|
||||
|
||||
from ..configs import InklingModelConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import (
|
||||
RoutedExperts,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gate / expert selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_INKLING_LL_BF16_MAX_TOKENS = 64
|
||||
_MXFP4_INPUT_SCALE_DENOMINATOR = torch.finfo(torch.float8_e4m3fn).max * 6.0
|
||||
|
||||
|
||||
def _linear_with_fp32_out(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
||||
leading = list(x.shape[:-1])
|
||||
flat = x.flatten(0, -2)
|
||||
if (
|
||||
flat.shape[0] <= _INKLING_LL_BF16_MAX_TOKENS
|
||||
and flat.dtype == torch.bfloat16
|
||||
and weight.dtype == torch.bfloat16
|
||||
and flat.is_cuda
|
||||
and flat.is_contiguous()
|
||||
and weight.is_contiguous()
|
||||
and flat.shape[1] % 8 == 0
|
||||
and current_platform.has_device_capability(90)
|
||||
and ll_bf16.is_available()
|
||||
):
|
||||
out = ll_bf16.ll_bf16_gemm(flat, weight)
|
||||
else:
|
||||
out = torch.mm(flat, weight.T, out_dtype=torch.float32)
|
||||
return out.view(*leading, weight.shape[0])
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["T", "route_scale"])
|
||||
def _inkling_gate_select_kernel(
|
||||
logits_ptr, # [T, G] fp32 gate logits (stride_logits_0 may include pad)
|
||||
bias_ptr, # [R] fp32 selection bias (or 0 ptr if HAS_BIAS=False)
|
||||
global_scale_ptr, # [1] fp32 (or unused if HAS_GSCALE=False)
|
||||
ids_ptr, # [T, K + S] int32 out: selected expert ids
|
||||
weights_ptr, # [T, K + S] fp32 out: renormalized weights
|
||||
route_scale,
|
||||
T,
|
||||
G: tl.constexpr, # total gate experts (routed + shared)
|
||||
stride_logits_0,
|
||||
R: tl.constexpr, # routed experts
|
||||
K: tl.constexpr, # top-k routed
|
||||
S: tl.constexpr, # shared (sink) experts
|
||||
HAS_BIAS: tl.constexpr,
|
||||
HAS_GSCALE: tl.constexpr,
|
||||
BLOCK_G: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0).to(tl.int64)
|
||||
if pid >= T:
|
||||
return
|
||||
offs = tl.arange(0, BLOCK_G)
|
||||
mask_r = offs < R
|
||||
logits = tl.load(
|
||||
logits_ptr + pid * stride_logits_0 + offs,
|
||||
mask=offs < G,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
|
||||
# Selection scores: sigmoid(routed logits) (+ bias), non-routed lanes -inf.
|
||||
sel = tl.where(mask_r, tl.sigmoid(logits), float("-inf"))
|
||||
if HAS_BIAS:
|
||||
bias = tl.load(bias_ptr + offs, mask=mask_r, other=0.0).to(tl.float32)
|
||||
sel = tl.where(mask_r, sel + bias, float("-inf"))
|
||||
|
||||
scale = route_scale
|
||||
if HAS_GSCALE:
|
||||
scale = scale * tl.load(global_scale_ptr).to(tl.float32)
|
||||
|
||||
# Iterative top-K (K is small); argmax tie-breaks to the lowest index
|
||||
# (stable ordering).
|
||||
A: tl.constexpr = K + S
|
||||
offs_a = tl.arange(0, A)
|
||||
top_ids = tl.zeros([A], dtype=tl.int32)
|
||||
active = tl.zeros([A], dtype=tl.float32)
|
||||
for kk in tl.static_range(K):
|
||||
idx = tl.argmax(sel, axis=0).to(tl.int32)
|
||||
raw = tl.max(tl.where(offs == idx, logits, float("-inf")), axis=0)
|
||||
top_ids = tl.where(offs_a == kk, idx, top_ids)
|
||||
active = tl.where(offs_a == kk, raw, active)
|
||||
sel = tl.where(offs == idx, float("-inf"), sel)
|
||||
if S > 0:
|
||||
# Shared sink logits sit at the tail of the gate output; their expert
|
||||
# ids continue after the routed range (R + j).
|
||||
for jj in tl.static_range(S):
|
||||
raw = tl.max(tl.where(offs == R + jj, logits, float("-inf")), axis=0)
|
||||
top_ids = tl.where(offs_a == K + jj, tl.full([], R + jj, tl.int32), top_ids)
|
||||
active = tl.where(offs_a == K + jj, raw, active)
|
||||
|
||||
# Log-sigmoid renormalization over the K + S active logits.
|
||||
abs_l = tl.abs(active)
|
||||
min_l = tl.minimum(active, 0.0)
|
||||
log_probs = min_l - tldevice.log1p(tldevice.exp(-abs_l))
|
||||
max_lp = tl.max(log_probs, axis=0)
|
||||
exp_shifted = tldevice.exp(log_probs - max_lp)
|
||||
sum_exp = tl.sum(exp_shifted, axis=0)
|
||||
weights = exp_shifted / sum_exp * scale
|
||||
|
||||
tl.store(ids_ptr + pid * A + offs_a, top_ids)
|
||||
tl.store(weights_ptr + pid * A + offs_a, weights)
|
||||
|
||||
|
||||
def inkling_gate_select(
|
||||
logits: torch.Tensor, # [T, >=G] fp32 (rows may carry GEMM padding)
|
||||
n_gate_experts: int,
|
||||
n_routed_experts: int,
|
||||
topk: int,
|
||||
n_shared_experts: int,
|
||||
bias: torch.Tensor | None,
|
||||
route_scale: float,
|
||||
global_scale: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Sigmoid + bias + top-k + log-sigmoid renorm; returns (weights, ids)."""
|
||||
assert logits.dtype == torch.float32
|
||||
tokens = logits.shape[0]
|
||||
active = topk + n_shared_experts
|
||||
topk_ids = torch.empty((tokens, active), dtype=torch.int32, device=logits.device)
|
||||
topk_weights = torch.empty(
|
||||
(tokens, active), dtype=torch.float32, device=logits.device
|
||||
)
|
||||
if tokens == 0:
|
||||
return topk_weights, topk_ids
|
||||
_inkling_gate_select_kernel[(tokens,)](
|
||||
logits,
|
||||
bias if bias is not None else logits,
|
||||
global_scale if global_scale is not None else logits,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
route_scale,
|
||||
tokens,
|
||||
n_gate_experts,
|
||||
logits.stride(0),
|
||||
n_routed_experts,
|
||||
topk,
|
||||
n_shared_experts,
|
||||
HAS_BIAS=bias is not None,
|
||||
HAS_GSCALE=global_scale is not None,
|
||||
BLOCK_G=triton.next_power_of_2(n_gate_experts),
|
||||
)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
class InklingGate(nn.Module):
|
||||
"""Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
|
||||
global scale (the served checkpoint's only configuration)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
d_model: int,
|
||||
n_routed_experts: int,
|
||||
n_shared_experts: int,
|
||||
experts_per_token: int,
|
||||
route_scale: float,
|
||||
*,
|
||||
use_global_scale: bool = False,
|
||||
use_gate_bias: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.n_routed_experts = n_routed_experts
|
||||
self.n_shared_experts = n_shared_experts
|
||||
self.n_total_experts = n_routed_experts + n_shared_experts
|
||||
self.topk = experts_per_token
|
||||
self.route_scale = route_scale
|
||||
|
||||
padded_experts = self.n_total_experts + (-self.n_total_experts) % 8
|
||||
self.weight = Parameter(
|
||||
torch.empty(padded_experts, d_model), requires_grad=False
|
||||
)
|
||||
set_weight_attrs(self.weight, {"weight_loader": self._load_weight})
|
||||
self.global_scale: Parameter | None
|
||||
if use_global_scale:
|
||||
self.global_scale = Parameter(
|
||||
torch.empty(1, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
else:
|
||||
self.global_scale = None
|
||||
self.bias: Parameter | None
|
||||
if use_gate_bias:
|
||||
self.bias = Parameter(
|
||||
torch.empty(n_routed_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
else:
|
||||
self.bias = None
|
||||
|
||||
@staticmethod
|
||||
def _load_weight(param: Parameter, loaded_weight: torch.Tensor) -> None:
|
||||
param.data.zero_()
|
||||
param.data[: loaded_weight.shape[0]].copy_(loaded_weight)
|
||||
|
||||
def compute_logits(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""fp32 gate logits [T, n_total_experts + pad] (pad columns are junk)."""
|
||||
return _linear_with_fp32_out(x, self.weight)
|
||||
|
||||
def select_experts(
|
||||
self, gating_output: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Full selection: (weights, ids) of [T, K + S]. The first K entries
|
||||
are the routed top-k; the S trailing entries are the sink gammas."""
|
||||
return inkling_gate_select(
|
||||
gating_output,
|
||||
self.n_total_experts,
|
||||
self.n_routed_experts,
|
||||
self.topk,
|
||||
self.n_shared_experts,
|
||||
self.bias,
|
||||
self.route_scale,
|
||||
self.global_scale,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MoE layer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _inkling_moe_ep_size() -> int:
|
||||
"""EP size the FusedMoE layer will run with (mirrors
|
||||
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
|
||||
expert parallelism is enabled)."""
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
if not parallel_config.enable_expert_parallel:
|
||||
return 1
|
||||
world = (
|
||||
get_tensor_model_parallel_world_size()
|
||||
* get_dp_group().world_size
|
||||
* get_pcp_group().world_size
|
||||
)
|
||||
return world if world > 1 else 1
|
||||
|
||||
|
||||
class InklingSinkExperts(nn.Module):
|
||||
"""Shared "sink" experts with per-token gammas, in bf16.
|
||||
|
||||
Replicated across EP ranks (every token activates every sink, so
|
||||
EP-sharding them would hotspot the owning rank) and TP-sharded on the
|
||||
intermediate dim so the output remains a TP-partial sum like the routed
|
||||
output. The sinks are always bf16 (the checkpoint excludes every
|
||||
``shared_experts`` from quantization): the experts concatenate into two
|
||||
plain dense GEMMs with the fused sink epilogue between them.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, n_experts: int, d_model: int, d_mlp: int, *, prefix: str = ""
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.n_experts = n_experts
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
intermediate_pp = d_mlp // tp_size
|
||||
self.w13_weight = Parameter(
|
||||
torch.empty(n_experts, 2 * intermediate_pp, d_model),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.w2_weight = Parameter(
|
||||
torch.empty(d_model, n_experts * intermediate_pp),
|
||||
requires_grad=False,
|
||||
)
|
||||
self._unit: torch.Tensor | None = None
|
||||
|
||||
def load_weight(self, key: str, weight: torch.Tensor) -> list[str]:
|
||||
"""Load one checkpoint sink tensor (stacked over the S experts)."""
|
||||
if key == "w13_weight":
|
||||
if weight.shape != self.w13_weight.shape:
|
||||
shard = self.w13_weight.shape[1]
|
||||
weight = weight.narrow(1, self.tp_rank * shard, shard)
|
||||
self.w13_weight.data.copy_(weight)
|
||||
return [key]
|
||||
|
||||
assert key == "w2_weight"
|
||||
shard = self.w2_weight.shape[1] // self.n_experts
|
||||
shard_start = 0 if weight.shape[2] == shard else self.tp_rank * shard
|
||||
for expert_idx, expert_weight in enumerate(weight):
|
||||
local_weight = expert_weight.narrow(1, shard_start, shard)
|
||||
start = expert_idx * shard
|
||||
self.w2_weight.data[:, start : start + shard].copy_(local_weight)
|
||||
return [key]
|
||||
|
||||
def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor:
|
||||
"""``sum_e gammas[:, e] * MLP_e(x)`` (TP-partial along d_mlp)."""
|
||||
from .ops import sink_silu_mul_epilogue
|
||||
|
||||
# One GEMM over the experts' stacked w13 (a view), fused epilogue,
|
||||
# then one GEMM whose K-reduction over the K-concatenated w2 performs
|
||||
# the expert sum.
|
||||
if self._unit is None or self._unit.device != x.device:
|
||||
self._unit = torch.ones(
|
||||
self.n_experts, dtype=torch.float32, device=x.device
|
||||
)
|
||||
raw = x @ self.w13_weight.view(-1, x.shape[-1]).T # (T, S*2F)
|
||||
h = sink_silu_mul_epilogue(
|
||||
raw, self._unit, gammas, self._unit, self.n_experts, x.dtype
|
||||
)
|
||||
return h @ self.w2_weight.T # (T, D)
|
||||
|
||||
|
||||
class InklingSinkExpertsLinear(nn.Module):
|
||||
"""LoRA-capable implementation of the Inkling sink experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
n_experts: int,
|
||||
d_model: int,
|
||||
d_mlp: int,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
|
||||
self.n_experts = n_experts
|
||||
self.d_mlp = d_mlp
|
||||
total = n_experts * d_mlp
|
||||
self.w13 = MergedColumnParallelLinear(
|
||||
input_size=d_model,
|
||||
output_sizes=[total, total],
|
||||
bias=False,
|
||||
prefix=f"{prefix}.w13",
|
||||
)
|
||||
self.w2 = RowParallelLinear(
|
||||
input_size=total,
|
||||
output_size=d_model,
|
||||
bias=False,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.w2",
|
||||
)
|
||||
self._w2_input_pp = self.w2.input_size_per_partition
|
||||
self._col_expert: torch.Tensor | None = None
|
||||
|
||||
def _gamma_expand(self, gammas: torch.Tensor) -> torch.Tensor:
|
||||
if self._col_expert is None or self._col_expert.device != gammas.device:
|
||||
local = self._w2_input_pp
|
||||
start = get_tensor_model_parallel_rank() * local
|
||||
cols = torch.arange(start, start + local, device=gammas.device)
|
||||
self._col_expert = (cols // self.d_mlp).long()
|
||||
return gammas[:, self._col_expert]
|
||||
|
||||
def load_weight(self, key: str, weight: torch.Tensor) -> list[str]:
|
||||
if key == "w13_weight":
|
||||
d_model = weight.shape[-1]
|
||||
gate = weight[:, 0::2, :].reshape(-1, d_model).contiguous()
|
||||
up = weight[:, 1::2, :].reshape(-1, d_model).contiguous()
|
||||
self.w13.weight_loader(self.w13.weight, gate, 0)
|
||||
self.w13.weight_loader(self.w13.weight, up, 1)
|
||||
return ["w13.weight"]
|
||||
w = weight.permute(1, 0, 2).reshape(weight.shape[1], -1).contiguous()
|
||||
self.w2.weight_loader(self.w2.weight, w)
|
||||
return ["w2.weight"]
|
||||
|
||||
def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor:
|
||||
gate_up, _ = self.w13(x)
|
||||
gate, up = gate_up.chunk(2, dim=-1)
|
||||
hidden_states = torch.nn.functional.silu(gate) * up
|
||||
hidden_states = (hidden_states * self._gamma_expand(gammas)).to(x.dtype)
|
||||
output, _ = self.w2(hidden_states)
|
||||
return output
|
||||
|
||||
|
||||
class InklingMoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: InklingModelConfig,
|
||||
*,
|
||||
prefix: str = "",
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Overfit to the served checkpoint: sigmoid gate renormalized after
|
||||
# top-k, shared sink experts, interleaved gate/up checkpoint rows.
|
||||
assert config.gate_activation == "sigmoid" and config.norm_after_topk
|
||||
assert config.n_shared_experts > 0 and config.shared_expert_sink
|
||||
assert config.inference_moe_w13_interleaved
|
||||
n_routed = config.n_routed_experts
|
||||
n_shared = config.n_shared_experts
|
||||
self.n_routed_experts = n_routed
|
||||
self.gate = InklingGate(
|
||||
d_model=config.hidden_size,
|
||||
n_routed_experts=n_routed,
|
||||
n_shared_experts=n_shared,
|
||||
experts_per_token=config.num_experts_per_tok,
|
||||
route_scale=config.route_scale,
|
||||
use_global_scale=config.use_global_scale,
|
||||
use_gate_bias=config.use_gate_bias,
|
||||
)
|
||||
|
||||
# TRTLLM MoE kernels assume equal, contiguous per-rank expert slabs
|
||||
# (local_expert_offset = ep_rank * local_num_experts), so pad the
|
||||
# expert count to a multiple of the EP size. A no-op for the usual
|
||||
# power-of-two EP sizes (n_routed is a power of two).
|
||||
num_experts = n_routed + (-n_routed) % _inkling_moe_ep_size()
|
||||
|
||||
# The released MXFP4 checkpoint keeps the first routed-expert layer
|
||||
# in bf16 and lists its two expert weights explicitly in Quark's
|
||||
# exclusion list. FusedMoE asks for a quant method at the module
|
||||
# prefix, so an exact weight exclusion would otherwise be missed and
|
||||
# the bf16 tensors would be loaded into MXFP4 parameters.
|
||||
routed_quant_config = quant_config
|
||||
if quant_config is not None:
|
||||
excluded = set(getattr(quant_config, "quant_config", {}).get("exclude", ()))
|
||||
expert_prefix = f"{prefix}.experts"
|
||||
routed_weights = {
|
||||
f"{expert_prefix}.w13_weight",
|
||||
f"{expert_prefix}.w2_weight",
|
||||
}
|
||||
if routed_weights <= excluded:
|
||||
routed_quant_config = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
num_experts=num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
renormalize=False,
|
||||
quant_config=routed_quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
custom_routing_function=self._select_routed,
|
||||
router_logits_dtype=torch.float32,
|
||||
activation="silu",
|
||||
)
|
||||
# The decoder layer reduce-scatters the MoE delta into the sconv
|
||||
# stream itself (RS -> shard sconv -> AG); the runner must return the
|
||||
# per-rank partial sum instead of all-reducing.
|
||||
self.experts.moe_config.skip_final_all_reduce = True
|
||||
|
||||
self._routed_sel: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None
|
||||
sink_experts_cls = (
|
||||
InklingSinkExpertsLinear
|
||||
if get_current_vllm_config().lora_config is not None
|
||||
else InklingSinkExperts
|
||||
)
|
||||
self.sink_experts = sink_experts_cls(
|
||||
n_experts=n_shared,
|
||||
d_model=config.hidden_size,
|
||||
d_mlp=config.intermediate_size,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
# Sink chain overlaps the routed MoE call on the aux stream for
|
||||
# decode-sized batches (same pattern as the runner's SharedExperts
|
||||
# multi-stream overlap). The routed GEMM runs on the default stream and
|
||||
# the sink chain on the aux stream, joined via these two events by
|
||||
# ``maybe_execute_in_parallel``.
|
||||
self._sink_stream: torch.cuda.Stream | None = aux_stream()
|
||||
self._sink_events = (torch.cuda.Event(), torch.cuda.Event())
|
||||
|
||||
def _select_routed(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""FusedMoE ``custom_routing_function``: the routed top-k slice of the
|
||||
full (routed + sink) selection.
|
||||
|
||||
forward() stashes its selection (keyed by logits identity) so the
|
||||
gate select runs once per layer; the fallback covers paths where the
|
||||
runner re-derives the logits (e.g. naive DP dispatch).
|
||||
"""
|
||||
del hidden_states, renormalize
|
||||
assert topk == self.gate.topk
|
||||
cached = self._routed_sel
|
||||
self._routed_sel = None
|
||||
if cached is not None and cached[0] is gating_output:
|
||||
return cached[1], cached[2]
|
||||
weights, ids = self.gate.select_experts(gating_output)
|
||||
return weights[:, :topk].contiguous(), ids[:, :topk].contiguous()
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor | None:
|
||||
router_logits = self.gate.compute_logits(x)
|
||||
num_tokens = x.shape[0]
|
||||
# One gate select per layer: the routed slice is stashed for the
|
||||
# routing function inside the FusedMoE op; the sink gammas are the
|
||||
# trailing columns.
|
||||
k = self.gate.topk
|
||||
weights, ids = self.gate.select_experts(router_logits)
|
||||
self._routed_sel = (
|
||||
router_logits,
|
||||
weights[:, :k].contiguous(),
|
||||
ids[:, :k].contiguous(),
|
||||
)
|
||||
gammas = weights[:, k:]
|
||||
|
||||
out, sink_out = maybe_execute_in_parallel(
|
||||
lambda: self.experts(hidden_states=x, router_logits=router_logits),
|
||||
lambda: self.sink_experts(x, gammas),
|
||||
self._sink_events[0],
|
||||
self._sink_events[1],
|
||||
self._sink_stream
|
||||
if num_tokens <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
||||
else None,
|
||||
)
|
||||
self._routed_sel = None
|
||||
|
||||
return out.add_(sink_out)
|
||||
|
||||
# -- weight loading ----------------------------------------------------
|
||||
|
||||
def _local_expert_slots(self) -> dict[int, int]:
|
||||
"""Global expert id -> local slot for this rank's expert partition."""
|
||||
manager = self.experts.routed_experts.expert_map_manager
|
||||
if manager.expert_map is None:
|
||||
return {g: g for g in range(manager.global_num_experts)}
|
||||
emap = manager.expert_map.tolist()
|
||||
return {g: slot for g, slot in enumerate(emap) if slot >= 0}
|
||||
|
||||
def load_expert_weight(self, name: str, weight: torch.Tensor) -> list[str]:
|
||||
"""Load one checkpoint expert tensor.
|
||||
|
||||
``name`` is relative to the mlp module: ``experts.<t>`` (routed
|
||||
stack) or ``shared_experts.shared_<t>`` (sink experts). Returns the
|
||||
loaded param names (relative to this module).
|
||||
"""
|
||||
if name.startswith("shared_experts."):
|
||||
key = name.split(".", 1)[1].replace("shared_", "", 1)
|
||||
return [
|
||||
f"sink_experts.{p}" for p in self.sink_experts.load_weight(key, weight)
|
||||
]
|
||||
|
||||
experts: RoutedExperts = self.experts.routed_experts
|
||||
key = name.split(".", 1)[1]
|
||||
|
||||
# original_shape is unused by the vLLM serving layout.
|
||||
if key.endswith(".original_shape"):
|
||||
return []
|
||||
if key.endswith(".input_amax"):
|
||||
projection = "w13" if key.startswith("w13") else "w2"
|
||||
amax = float(weight.max())
|
||||
assert math.isfinite(amax) and amax > 0, (
|
||||
f"bad {projection} input_amax: {amax}"
|
||||
)
|
||||
input_scale = getattr(experts, f"{projection}_input_scale")
|
||||
input_scale.data.fill_(amax / _MXFP4_INPUT_SCALE_DENOMINATOR)
|
||||
return [f"experts.routed_experts.{projection}_input_scale"]
|
||||
|
||||
# Quark's OCP MXFP4 converter stores block scales as a two-dimensional
|
||||
# tensor with the expert and projection-row dimensions flattened:
|
||||
# w13: [E * 2I, H / 32], w2: [E * H, I / 32].
|
||||
# Restore the expert dimension before applying the EP/TP slicing used
|
||||
# for both packed weights and scales.
|
||||
if key.endswith("_weight_scale") and weight.ndim == 2:
|
||||
if weight.shape[0] % self.n_routed_experts != 0:
|
||||
raise ValueError(
|
||||
f"cannot unflatten {name} with shape {tuple(weight.shape)} "
|
||||
f"over {self.n_routed_experts} experts"
|
||||
)
|
||||
weight = weight.view(
|
||||
self.n_routed_experts,
|
||||
weight.shape[0] // self.n_routed_experts,
|
||||
weight.shape[1],
|
||||
)
|
||||
|
||||
param = getattr(experts, key)
|
||||
slots = self._local_expert_slots()
|
||||
gids = sorted(slots)
|
||||
lids = [slots[g] for g in gids]
|
||||
tp_rank = experts.moe_config.moe_parallel_config.tp_rank
|
||||
tp_size = experts.moe_config.moe_parallel_config.tp_size
|
||||
|
||||
if key.endswith("_scale_2"):
|
||||
# Per-expert scalars, vectorized over the local experts. The
|
||||
# fused w13 param carries one slot per gate/up half.
|
||||
vals = weight[gids].float().to(param.device)
|
||||
param.data[lids] = vals[:, None] if param.data.ndim == 2 else vals
|
||||
elif key.startswith("w13"):
|
||||
# Checkpoint w13 rows are interleaved [g0, u0, g1, u1, ...]; the
|
||||
# fused param layout is [w1(gate); w3(up)]. The TP-local rows form
|
||||
# one contiguous slab of the interleaved tensor, so upload just
|
||||
# that slab (a single bounded synchronous H2D; pre-uploading whole
|
||||
# untrimmed tensors pins the mmap pages of the entire checkpoint
|
||||
# and OOMs the host) and de-interleave on device.
|
||||
dst_half = param.shape[1] // 2
|
||||
if weight.shape[1] % (2 * tp_size) != 0:
|
||||
raise ValueError(
|
||||
f"cannot TP-shard {name} with shape {tuple(weight.shape)} "
|
||||
f"over {tp_size} ranks"
|
||||
)
|
||||
logical_half = weight.shape[1] // (2 * tp_size)
|
||||
if logical_half > dst_half:
|
||||
raise ValueError(
|
||||
f"checkpoint shard for {name} has {logical_half} rows per "
|
||||
f"gate/up half, but destination only has {dst_half}"
|
||||
)
|
||||
for gid, lid in slots.items():
|
||||
slab = weight[gid].narrow(
|
||||
0,
|
||||
tp_rank * 2 * logical_half,
|
||||
2 * logical_half,
|
||||
)
|
||||
slab = slab.to(param.device)
|
||||
param.data[lid, :logical_half].copy_(slab[0::2])
|
||||
param.data[
|
||||
lid,
|
||||
dst_half : dst_half + logical_half,
|
||||
].copy_(slab[1::2])
|
||||
else:
|
||||
# w2: shard the packed intermediate (last) dim. AITER rounds the
|
||||
# destination intermediate width to 256, so derive the logical TP
|
||||
# slice from the checkpoint and leave the destination tail at its
|
||||
# initialized padding value (zero for weights, one for scales).
|
||||
if weight.shape[2] % tp_size != 0:
|
||||
raise ValueError(
|
||||
f"cannot TP-shard {name} with shape {tuple(weight.shape)} "
|
||||
f"over {tp_size} ranks"
|
||||
)
|
||||
shard = weight.shape[2] // tp_size
|
||||
if shard > param.shape[2]:
|
||||
raise ValueError(
|
||||
f"checkpoint shard for {name} has width {shard}, but "
|
||||
f"destination only has {param.shape[2]}"
|
||||
)
|
||||
for gid, lid in slots.items():
|
||||
param.data[lid, :, :shard].copy_(
|
||||
weight[gid].narrow(1, tp_rank * shard, shard)
|
||||
)
|
||||
return [f"experts.routed_experts.{key}"]
|
||||
|
||||
def finalize_load(self) -> list[str]:
|
||||
"""Post-load fixups for zeroed padding experts."""
|
||||
experts = self.experts.routed_experts
|
||||
out: list[str] = []
|
||||
# Zero the EP-alignment padding experts (if any) so their
|
||||
# (never-routed) slots hold defined values.
|
||||
slots = self._local_expert_slots()
|
||||
for gid in range(self.n_routed_experts, experts.global_num_experts):
|
||||
lid = slots.get(gid)
|
||||
if lid is None:
|
||||
continue
|
||||
for pname in (
|
||||
"w13_weight",
|
||||
"w2_weight",
|
||||
"w13_weight_scale",
|
||||
"w2_weight_scale",
|
||||
"w13_weight_scale_2",
|
||||
"w2_weight_scale_2",
|
||||
):
|
||||
p = getattr(experts, pname, None)
|
||||
if p is not None:
|
||||
p.data[lid].zero_()
|
||||
return out
|
||||
@@ -0,0 +1,410 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling MTP (Multi-Token Prediction) draft model (NVIDIA).
|
||||
|
||||
Implements the first MTP depth from the reference ``mtp_model.py`` shipped with
|
||||
the checkpoint. It owns ``hidden_norm`` / ``embed_norm`` RMSNorms, a ``2H -> H``
|
||||
input projection, and a full Inkling transformer block with a dense bf16 MLP.
|
||||
|
||||
The draft shares the target's token embedding table and LM head
|
||||
(``load_eagle_model`` wires those references) and applies the backbone
|
||||
``embed_norm`` on top: the depth layers were trained on the same normed
|
||||
embeddings the backbone consumes (their own ``embed_norm`` weights are
|
||||
near-identity trims, unlike the backbone's whitening ``embed_norm``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..configs import InklingModelConfig
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
|
||||
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
|
||||
|
||||
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
|
||||
# Mirrors the backbone's hf_to_vllm_mapper.orig_to_new_stacked; kept as a
|
||||
# local (pname, wname, shard) list since the MTP loader remaps by hand.
|
||||
_ATTENTION_PARAMS_MAPPING = [
|
||||
("qkvr", "wq_du", 0),
|
||||
("qkvr", "wk_dv", 1),
|
||||
("qkvr", "wv_dv", 2),
|
||||
("qkvr", "wr_du", 3),
|
||||
]
|
||||
|
||||
|
||||
def _mtp_depth_from_name(name: str) -> int | None:
|
||||
m = re.search(r"\.mtp\.layers\.(\d+)\.", name)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
class InklingMTPDepthLayer(nn.Module):
|
||||
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
|
||||
|
||||
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.input_proj = ReplicatedLinear(
|
||||
config.hidden_size * 2,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
prefix=f"{prefix}.input_proj",
|
||||
)
|
||||
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
|
||||
# full attention (the swa_* head config and sliding_window window) to
|
||||
# match this depth's checkpoint transformer_block weights.
|
||||
self.transformer_block = InklingDecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
is_local=is_local,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.transformer_block",
|
||||
force_dense_mlp=True,
|
||||
)
|
||||
|
||||
def forward(self, combined: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
# ``combined`` is the fused-normed [rmsnorm(hidden) | embed_norm(emb)]
|
||||
# input, built by InklingMultiTokenPredictor.fused_input_cat in one launch.
|
||||
hidden = self.input_proj(combined)
|
||||
# The short conv self-fetches its paged SWA-cache metadata from the
|
||||
# forward context (via its conv_owner prefix); no conv_meta to thread.
|
||||
return self.transformer_block(positions, hidden)
|
||||
|
||||
|
||||
class InklingMultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
config: InklingModelConfig = (
|
||||
vllm_config.speculative_config.draft_model_config.hf_config
|
||||
)
|
||||
self.config = config
|
||||
if vllm_config.speculative_config.num_speculative_tokens != 1:
|
||||
raise ValueError(
|
||||
"Inkling MTP currently supports exactly one speculative token"
|
||||
)
|
||||
self.chain_hidden_post_norm = config.chain_hidden_post_norm
|
||||
local_ids = set(config.local_layer_ids)
|
||||
self.layers = nn.ModuleDict(
|
||||
{"0": InklingMTPDepthLayer(config, f"{prefix}.layers.0", 0 in local_ids)}
|
||||
)
|
||||
self.chain_norm = (
|
||||
InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
if self.chain_hidden_post_norm
|
||||
else None
|
||||
)
|
||||
# The target's raw token embedding (pre embed_norm), attached by
|
||||
# load_eagle_model. Never materialized here: building our own
|
||||
# replicated copy would transiently double the 2.3 GiB table.
|
||||
self.embed_tokens: InklingReplicatedEmbedding = None # type: ignore[assignment]
|
||||
# The depth layers consume the *backbone-normed* embedding
|
||||
# (embed_norm(embed(ids))), not the raw one: mtp embed_norm weights
|
||||
# are near-identity (trained on already-normalized inputs), and
|
||||
# feeding raw embeddings drops MTP1 acceptance from ~0.85 to ~0.70.
|
||||
# Weight loaded from the target's embed_norm.weight; gated like the
|
||||
# target's InklingModel.embed_norm.
|
||||
self.backbone_embed_norm = (
|
||||
InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
if config.use_embed_norm
|
||||
else None
|
||||
)
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: object | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Draft-prefill embedding: fused gather + backbone embed_norm, then
|
||||
the target's tower embeddings scattered in unnormed (the backbone
|
||||
convention — MM embeds are merged after embed_norm)."""
|
||||
norm = self.backbone_embed_norm
|
||||
embeds = embed_rmsnorm(
|
||||
input_ids,
|
||||
self.embed_tokens.weight,
|
||||
norm.weight if norm is not None else None,
|
||||
norm.variance_epsilon if norm is not None else 0.0,
|
||||
)
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0: # type: ignore[arg-type]
|
||||
return embeds
|
||||
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
|
||||
|
||||
assert is_multimodal is not None
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def fused_input_cat(
|
||||
self,
|
||||
layer: InklingMTPDepthLayer,
|
||||
previous_hidden: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one
|
||||
launch: embedding row gather + the backbone embed_norm + the depth
|
||||
embed_norm chain on one side, hidden_norm on the other, written
|
||||
straight into the cat buffer."""
|
||||
hidden_w = layer.hidden_norm.weight
|
||||
embed_w = layer.embed_norm.weight
|
||||
eps = layer.hidden_norm.variance_epsilon
|
||||
if inputs_embeds is not None:
|
||||
# Draft prefill with target-merged MM embeddings (already
|
||||
# backbone-normed via embed_input_ids); only the depth embed_norm
|
||||
# remains.
|
||||
return embed_dual_rmsnorm_cat(
|
||||
previous_hidden, hidden_w, embed_w, eps, embeds=inputs_embeds
|
||||
)
|
||||
return embed_dual_rmsnorm_cat(
|
||||
previous_hidden,
|
||||
hidden_w,
|
||||
embed_w,
|
||||
eps,
|
||||
input_ids=input_ids,
|
||||
embed_table=self.embed_tokens.weight,
|
||||
pre_norm_weight=(
|
||||
self.backbone_embed_norm.weight
|
||||
if self.backbone_embed_norm is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
# The draft's short conv is a paged SWA-cache layer (its conv_owner is
|
||||
# auto-enumerated as a draft attention layer); its per-token metadata is
|
||||
# built by the speculator's build_attn_metadata and read from the
|
||||
# forward context, so nothing extra is threaded here.
|
||||
if spec_step_idx != 0:
|
||||
raise ValueError("Inkling MTP only supports spec_step_idx=0")
|
||||
layer = self.layers["0"]
|
||||
combined = self.fused_input_cat(
|
||||
layer, previous_hidden_states, input_ids, inputs_embeds
|
||||
)
|
||||
hidden = layer(combined, positions)
|
||||
if self.chain_norm is not None:
|
||||
hidden = self.chain_norm(hidden)
|
||||
return hidden
|
||||
|
||||
|
||||
class InklingMTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
config: InklingModelConfig = (
|
||||
vllm_config.speculative_config.draft_model_config.hf_config
|
||||
)
|
||||
self.config = config
|
||||
self.model = InklingMultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
# The target's (vocab-sharded) LM head, attached by load_eagle_model;
|
||||
# never materialized here (same reasoning as model.embed_tokens).
|
||||
self.lm_head: ParallelLMHead = None # type: ignore[assignment]
|
||||
self.logits_processor = LogitsProcessor(
|
||||
config.padded_vocab_size,
|
||||
org_vocab_size=config.vocab_size,
|
||||
soft_cap=config.final_logit_softcapping,
|
||||
)
|
||||
self._logits_zero: torch.Tensor | None = None
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: object | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(
|
||||
input_ids, multimodal_embeddings, is_multimodal=is_multimodal
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
hidden_states,
|
||||
inputs_embeds,
|
||||
spec_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
# The MTP shares the base model's LM head, which is trained on
|
||||
# ``hidden / mup``-scaled inputs, so apply the same mup scaling here
|
||||
# for a matching logit scale — folded into the lm_head GEMM alpha
|
||||
# (fp32 epilogue) like the target's compute_logits. (Argmax-invariant
|
||||
# for greedy draft sampling, but it matters for the gumbel sampling
|
||||
# distribution at temperature > 0.)
|
||||
mup = self.config.logits_mup_width_multiplier
|
||||
if not mup:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
assert self.logits_processor.soft_cap is None
|
||||
assert self.logits_processor.scale == 1.0
|
||||
w = self.lm_head.weight
|
||||
if self._logits_zero is None:
|
||||
self._logits_zero = w.new_zeros(1)
|
||||
logits = torch.addmm(
|
||||
self._logits_zero,
|
||||
hidden_states,
|
||||
w.t(),
|
||||
beta=0.0,
|
||||
alpha=1.0 / mup,
|
||||
)
|
||||
logits = self.logits_processor._gather_logits(logits)
|
||||
if logits is not None:
|
||||
logits = logits[..., : self.logits_processor.org_vocab_size]
|
||||
return logits
|
||||
|
||||
def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
|
||||
reduction — no full-vocab logits all-gather. The muP divisor is a
|
||||
positive scalar, so the argmax is invariant and the scaling is
|
||||
skipped entirely."""
|
||||
return self.logits_processor.get_top_tokens(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
return _load_inkling_mtp_weights(self, weights)
|
||||
|
||||
|
||||
def _load_inkling_mtp_weights(
|
||||
module: InklingMTP,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
"""Load ``model.mtp.*`` weights into the MTP module.
|
||||
|
||||
Checkpoint keys look like ``model.mtp.chain_norm.weight`` and
|
||||
``model.mtp.layers.{i}.{...}``. The transformer block reuses the backbone
|
||||
layer's fused-projection layout, so we apply the same qkvr / gate_up / down
|
||||
remapping as ``_load_inkling_weights``. Token embedding and LM head are shared
|
||||
(provided by ``load_eagle_model``) and are not present in mtp.safetensors.
|
||||
"""
|
||||
# Per-depth attention is full or sliding-window (config.local_layer_ids);
|
||||
# each depth's qkvr MergedColumnParallelLinear is built with the matching
|
||||
# (swa_)num_key_value_heads, and its weight_loader handles the TP sharding.
|
||||
# The sconv SWA cache pins tp_size <= num_key_value_heads, so tp never
|
||||
# exceeds a layer's kv-head count and no GQA K/V replication is needed here.
|
||||
params = dict(module.named_parameters())
|
||||
loaded: set[str] = set()
|
||||
|
||||
def _load(name: str, weight: torch.Tensor, shard_id: object = None) -> bool:
|
||||
param = params.get(name)
|
||||
if param is None:
|
||||
return False
|
||||
loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if shard_id is None:
|
||||
if loader is default_weight_loader or param.shape == weight.shape:
|
||||
default_weight_loader(param, weight)
|
||||
else:
|
||||
loader(param, weight)
|
||||
else:
|
||||
loader(param, weight, shard_id) # type: ignore[call-arg]
|
||||
loaded.add(name)
|
||||
return True
|
||||
|
||||
for name, weight in weights:
|
||||
depth = _mtp_depth_from_name(name)
|
||||
# Token embedding and LM head are never materialized on the draft
|
||||
# (no params to load into); load_eagle_model attaches the target's.
|
||||
if name in ("model.llm.embed.weight", "model.llm.unembed.weight"):
|
||||
continue
|
||||
# The backbone embed_norm, applied to the shared embedding before the
|
||||
# depth layers (see InklingMultiTokenPredictor.embed_input_ids). The
|
||||
# per-depth mtp.layers.{i}.embed_norm keys carry ".mtp." and are loaded
|
||||
# below. Only the shared backbone key routes here.
|
||||
if name == "model.llm.embed_norm.weight":
|
||||
_load("model.backbone_embed_norm.weight", weight)
|
||||
continue
|
||||
# Only consume the MTP weights; everything else belongs to the target.
|
||||
if ".mtp." not in name:
|
||||
continue
|
||||
# Only the first checkpoint depth is used for MTP=1.
|
||||
if depth is not None and depth != 0:
|
||||
continue
|
||||
# model.mtp.chain_norm.weight -> model.chain_norm.weight
|
||||
# model.mtp.layers.{i}.X -> model.layers.{i}.X
|
||||
original_name = name
|
||||
name = name.replace(".mtp.layers.", ".layers.").replace(
|
||||
".mtp.chain_norm.", ".chain_norm."
|
||||
)
|
||||
|
||||
if ".chain_norm." in name and module.model.chain_norm is None:
|
||||
raise ValueError(
|
||||
"Inkling checkpoint contains chain_norm weights but "
|
||||
"chain_hidden_post_norm is disabled."
|
||||
)
|
||||
|
||||
# Fused attention qkvr (wq_du/wk_dv/wv_dv/wr_du -> qkvr).
|
||||
matched = False
|
||||
for pname, wname, shard in _ATTENTION_PARAMS_MAPPING:
|
||||
if f".attn.{wname}." in name:
|
||||
mapped_name = name.replace(f".{wname}.", f".{pname}.")
|
||||
if not _load(mapped_name, weight, shard):
|
||||
raise ValueError(f"Unexpected Inkling MTP weight: {original_name}")
|
||||
matched = True
|
||||
break
|
||||
if matched:
|
||||
continue
|
||||
|
||||
# Dense MLP fused gate/up + down.
|
||||
if ".mlp.w13_dn.weight" in name:
|
||||
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
|
||||
elif ".mlp.w2_md.weight" in name:
|
||||
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params:
|
||||
continue
|
||||
loaded_weight = _load(name, weight)
|
||||
if not loaded_weight:
|
||||
raise ValueError(f"Unexpected Inkling MTP weight: {original_name}")
|
||||
required = {
|
||||
name
|
||||
for name in params
|
||||
if name.startswith("model.layers.") or name.startswith("model.chain_norm.")
|
||||
}
|
||||
if (missing := sorted(required - loaded)) and is_mtp_completeness_check_enabled():
|
||||
raise ValueError(
|
||||
"Inkling MTP checkpoint is missing required parameters: "
|
||||
+ ", ".join(missing)
|
||||
)
|
||||
return loaded
|
||||
|
||||
|
||||
EntryClass = [InklingMTP]
|
||||
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling kernels (NVIDIA).
|
||||
|
||||
``rmsnorm`` / ``sconv`` import eagerly. The SwiGLU kernels and the FA4
|
||||
relative-attention wrapper are exposed lazily to keep this package's
|
||||
import path lightweight.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from .norm import add_rmsnorm, rmsnorm
|
||||
from .sconv import fused_sconv
|
||||
|
||||
_LAZY_EXPORTS = {
|
||||
"silu_and_mul_triton": "silu_and_mul",
|
||||
"sink_silu_mul_epilogue": "silu_and_mul",
|
||||
"inkling_fa4_rel_attention": "fa4_rel_attention",
|
||||
"inkling_rel_attention_split_kv_decode": "rel_attention_decode",
|
||||
}
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .fa4_rel_attention import inkling_fa4_rel_attention # noqa: F401
|
||||
from .silu_and_mul import ( # noqa: F401
|
||||
silu_and_mul_triton,
|
||||
sink_silu_mul_epilogue,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"add_rmsnorm",
|
||||
"rmsnorm",
|
||||
"fused_sconv",
|
||||
*sorted(_LAZY_EXPORTS),
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
module = _LAZY_EXPORTS.get(name)
|
||||
if module is not None:
|
||||
import importlib
|
||||
|
||||
mod = importlib.import_module(f".{module}", __name__)
|
||||
return getattr(mod, name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,438 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm paged attention with Inkling's query-dependent relative bias.
|
||||
|
||||
The NVIDIA implementation uses the score-mod hook in tml-fa4. ROCm Flash
|
||||
Attention and AITER do not expose an equivalent hook, so this module implements
|
||||
the same operation directly in Triton. Query heads belonging to one KV head
|
||||
are processed together and KV pages are gathered through vLLM's block table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.inkling.amd.ops.rel_attention_decode import (
|
||||
inkling_rel_attention_split_kv_decode,
|
||||
use_split_kv_decode,
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def bucket_max_seqlen_q(max_seqlen_q: int) -> int:
|
||||
"""Round the scheduling bound up to a power of two."""
|
||||
return 1 << max(0, max_seqlen_q - 1).bit_length()
|
||||
|
||||
|
||||
def use_gfx950_gluon_decode(
|
||||
*, max_query_len: int, page_size: int, head_dim: int
|
||||
) -> bool:
|
||||
"""Use the vendored TokenSpeed CDNA4 decode kernel where it is supported."""
|
||||
return (
|
||||
os.getenv("INKLING_GFX950_GLUON", "1") == "1"
|
||||
and on_gfx950()
|
||||
and max_query_len == 1
|
||||
and page_size in (64, 128, 256)
|
||||
and head_dim in (64, 128)
|
||||
)
|
||||
|
||||
|
||||
def use_gfx950_gluon_extend(
|
||||
*,
|
||||
max_query_len: int,
|
||||
max_kv_len: int,
|
||||
page_size: int,
|
||||
head_dim: int,
|
||||
window_left: int,
|
||||
) -> bool:
|
||||
"""Use Gluon only for the long full-attention extend regime it wins."""
|
||||
return (
|
||||
os.getenv("INKLING_GFX950_GLUON", "1") == "1"
|
||||
and on_gfx950()
|
||||
and max_query_len > 1
|
||||
and max_kv_len >= 8192
|
||||
and window_left < 0
|
||||
and page_size in (64, 128, 256)
|
||||
and head_dim in (64, 128)
|
||||
)
|
||||
|
||||
|
||||
def inkling_fa4_num_splits(
|
||||
*,
|
||||
is_local: bool,
|
||||
batch_size: int,
|
||||
max_query_len: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
max_kv_len: int,
|
||||
) -> int:
|
||||
"""Keep the NVIDIA-facing split heuristic as API-compatible metadata.
|
||||
|
||||
The ROCm Triton implementation performs online softmax in one program and
|
||||
does not consume the result. Keeping this function unchanged avoids
|
||||
platform-specific scheduling branches in :mod:`inkling.amd.attention`.
|
||||
"""
|
||||
if is_local:
|
||||
return 1
|
||||
|
||||
q_rows = max_query_len * (num_heads // num_kv_heads)
|
||||
q_tiles = (q_rows + 255) // 256
|
||||
base_ctas = batch_size * num_kv_heads * q_tiles
|
||||
target_ctas = (
|
||||
256 if q_tiles == 1 and batch_size == 1 else (128 if q_tiles == 1 else 64)
|
||||
)
|
||||
max_splits = 128
|
||||
if q_tiles == 1 and batch_size == 1:
|
||||
if num_kv_heads == 8:
|
||||
max_splits = 16
|
||||
elif num_kv_heads == 4 or max_kv_len <= 8192:
|
||||
max_splits = 32
|
||||
elif max_kv_len <= 65536:
|
||||
max_splits = 64
|
||||
return max(
|
||||
1,
|
||||
min(target_ctas // base_ctas, max_splits, (max_kv_len + 127) // 128),
|
||||
)
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"BLOCK_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
||||
"BLOCK_H": lambda args: triton.next_power_of_2(args["gqa_group_size"]),
|
||||
"BLOCK_QH": lambda args: (
|
||||
args["BLOCK_Q"] * triton.next_power_of_2(args["gqa_group_size"])
|
||||
),
|
||||
}
|
||||
)
|
||||
@triton.jit(do_not_specialize_on_alignment=["cache_seqlens", "cu_seqlens_q"])
|
||||
def _inkling_rel_attention_kernel(
|
||||
q_ptr, # [total_q, Hq, D]
|
||||
k_ptr, # [blocks, page, Hkv, D]
|
||||
v_ptr, # [blocks, page, Hkv, D]
|
||||
rel_ptr, # [total_q, Hq, rel_extent]
|
||||
out_ptr, # [total_q, Hq, D]
|
||||
block_table_ptr, # [batch, max_pages]
|
||||
cache_seqlens,
|
||||
cu_seqlens_q,
|
||||
gqa_group_size,
|
||||
head_dim,
|
||||
softmax_scale,
|
||||
stride_q_t,
|
||||
stride_q_h,
|
||||
stride_q_d,
|
||||
stride_k_b,
|
||||
stride_k_p,
|
||||
stride_k_h,
|
||||
stride_k_d,
|
||||
stride_v_b,
|
||||
stride_v_p,
|
||||
stride_v_h,
|
||||
stride_v_d,
|
||||
stride_r_t,
|
||||
stride_r_h,
|
||||
stride_r_e,
|
||||
stride_o_t,
|
||||
stride_o_h,
|
||||
stride_o_d,
|
||||
stride_bt_b,
|
||||
page_size: tl.constexpr,
|
||||
rel_extent: tl.constexpr,
|
||||
window_left: tl.constexpr,
|
||||
BLOCK_Q: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
BLOCK_QH: tl.constexpr,
|
||||
):
|
||||
# A program owns BLOCK_Q query positions and every query head sharing one
|
||||
# KV head. This makes the QK/PV operations MFMA-friendly on CDNA.
|
||||
pid_q = tl.program_id(0)
|
||||
pid_kh = tl.program_id(1)
|
||||
pid_b = tl.program_id(2)
|
||||
|
||||
q_start = tl.load(cu_seqlens_q + pid_b)
|
||||
q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start
|
||||
q_block = pid_q * BLOCK_Q
|
||||
if q_block >= q_len:
|
||||
return
|
||||
|
||||
kv_len = tl.load(cache_seqlens + pid_b)
|
||||
prefix_len = kv_len - q_len
|
||||
q_head_start = pid_kh * gqa_group_size
|
||||
bt_row = block_table_ptr + pid_b * stride_bt_b
|
||||
|
||||
q_block_ptr = tl.make_block_ptr(
|
||||
base=q_ptr + q_start * stride_q_t + q_head_start * stride_q_h,
|
||||
shape=(q_len, gqa_group_size, head_dim),
|
||||
strides=(stride_q_t, stride_q_h, stride_q_d),
|
||||
offsets=(q_block, 0, 0),
|
||||
block_shape=(BLOCK_Q, BLOCK_H, BLOCK_D),
|
||||
order=(2, 1, 0),
|
||||
)
|
||||
q = tl.load(q_block_ptr, boundary_check=(0, 1, 2), padding_option="zero")
|
||||
q = tl.reshape(q, (BLOCK_QH, BLOCK_D))
|
||||
|
||||
q_rows = q_block + tl.arange(0, BLOCK_Q)
|
||||
q_abs = prefix_len + q_rows
|
||||
q_valid = q_rows < q_len
|
||||
off_d = tl.arange(0, BLOCK_D)
|
||||
d_valid = off_d < head_dim
|
||||
|
||||
m_i = tl.full((BLOCK_QH,), float("-inf"), dtype=tl.float32)
|
||||
l_i = tl.zeros((BLOCK_QH,), dtype=tl.float32)
|
||||
acc = tl.zeros((BLOCK_QH, BLOCK_D), dtype=tl.float32)
|
||||
log2e: tl.constexpr = 1.4426950408889634
|
||||
|
||||
for k_start in range(0, kv_len, BLOCK_K):
|
||||
k_pos = k_start + tl.arange(0, BLOCK_K)
|
||||
k_valid = k_pos < kv_len
|
||||
# Masked lanes still need an in-range page-table load.
|
||||
safe_pos = tl.minimum(k_pos, tl.maximum(kv_len - 1, 0))
|
||||
page = tl.load(bt_row + safe_pos // page_size).to(tl.int64)
|
||||
page_off = safe_pos % page_size
|
||||
|
||||
k = tl.load(
|
||||
k_ptr
|
||||
+ page[None, :] * stride_k_b
|
||||
+ page_off[None, :] * stride_k_p
|
||||
+ pid_kh * stride_k_h
|
||||
+ off_d[:, None] * stride_k_d,
|
||||
mask=d_valid[:, None] & k_valid[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
scores = tl.dot(q, k) * (softmax_scale * log2e)
|
||||
|
||||
dist = q_abs[:, None] - k_pos[None, :]
|
||||
score_valid = q_valid[:, None] & k_valid[None, :] & (dist >= 0)
|
||||
if window_left >= 0:
|
||||
score_valid &= dist <= window_left
|
||||
|
||||
# rel_logits is query- and head-dependent. Expand [Q, K] distance
|
||||
# indices across the GQA heads, then flatten to match QK's row order.
|
||||
off_h = tl.arange(0, BLOCK_H)
|
||||
rel_dist = dist[:, None, :]
|
||||
rel_valid = (
|
||||
q_valid[:, None, None]
|
||||
& (off_h[None, :, None] < gqa_group_size)
|
||||
& (rel_dist >= 0)
|
||||
& (rel_dist < rel_extent)
|
||||
)
|
||||
safe_dist = tl.maximum(0, tl.minimum(rel_dist, rel_extent - 1))
|
||||
bias = tl.load(
|
||||
rel_ptr
|
||||
+ (q_start + q_rows[:, None, None]) * stride_r_t
|
||||
+ (q_head_start + off_h[None, :, None]) * stride_r_h
|
||||
+ safe_dist * stride_r_e,
|
||||
mask=rel_valid,
|
||||
other=0.0,
|
||||
)
|
||||
bias = tl.reshape(bias, (BLOCK_QH, BLOCK_K))
|
||||
scores += bias.to(tl.float32) * log2e
|
||||
score_valid = tl.reshape(
|
||||
score_valid[:, None, :] & (off_h[None, :, None] < gqa_group_size),
|
||||
(BLOCK_QH, BLOCK_K),
|
||||
)
|
||||
scores = tl.where(score_valid, scores, float("-inf"))
|
||||
|
||||
m_ij = tl.maximum(m_i, tl.max(scores, axis=1))
|
||||
has_scores = tl.sum(score_valid.to(tl.int32), axis=1) > 0
|
||||
# A sliding-window query can have entire early KV tiles masked. Avoid
|
||||
# the -inf - -inf NaN in online softmax until its first live tile.
|
||||
alpha = tl.where(has_scores, tl.exp2(m_i - m_ij), 1.0)
|
||||
p = tl.where(
|
||||
score_valid,
|
||||
tl.exp2(scores - m_ij[:, None]),
|
||||
0.0,
|
||||
)
|
||||
l_i = l_i * alpha + tl.sum(p, axis=1)
|
||||
acc *= alpha[:, None]
|
||||
|
||||
v = tl.load(
|
||||
v_ptr
|
||||
+ page[:, None] * stride_v_b
|
||||
+ page_off[:, None] * stride_v_p
|
||||
+ pid_kh * stride_v_h
|
||||
+ off_d[None, :] * stride_v_d,
|
||||
mask=k_valid[:, None] & d_valid[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
acc += tl.dot(p.to(v.dtype), v)
|
||||
m_i = m_ij
|
||||
|
||||
acc /= l_i[:, None]
|
||||
acc = tl.reshape(acc, (BLOCK_Q, BLOCK_H, BLOCK_D))
|
||||
out_block_ptr = tl.make_block_ptr(
|
||||
base=out_ptr + q_start * stride_o_t + q_head_start * stride_o_h,
|
||||
shape=(q_len, gqa_group_size, head_dim),
|
||||
strides=(stride_o_t, stride_o_h, stride_o_d),
|
||||
offsets=(q_block, 0, 0),
|
||||
block_shape=(BLOCK_Q, BLOCK_H, BLOCK_D),
|
||||
order=(2, 1, 0),
|
||||
)
|
||||
tl.store(
|
||||
out_block_ptr,
|
||||
acc.to(out_ptr.dtype.element_ty),
|
||||
boundary_check=(0, 1, 2),
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def inkling_fa4_rel_attention(
|
||||
q: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
*,
|
||||
block_table: torch.Tensor,
|
||||
cache_seqlens: torch.Tensor,
|
||||
cu_seqlens_q: torch.Tensor,
|
||||
max_seqlen_q: int,
|
||||
softmax_scale: float,
|
||||
causal: bool,
|
||||
window_size: tuple[int, int],
|
||||
rel_extent: int,
|
||||
rel_logits: torch.Tensor,
|
||||
num_splits: int = 32,
|
||||
max_kv_len: int | None = None,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Paged varlen attention with Inkling's relative score modification."""
|
||||
del num_splits
|
||||
if not causal or window_size[1] not in (0, -1):
|
||||
raise NotImplementedError("Inkling ROCm attention requires causal masking")
|
||||
if q.ndim != 3 or key_cache.ndim != 4 or value_cache.ndim != 4:
|
||||
raise ValueError("expected q [T,H,D] and paged K/V [B,P,Hkv,D]")
|
||||
if rel_logits.shape != (q.shape[0], q.shape[1], rel_extent):
|
||||
raise ValueError(
|
||||
f"relative logits have shape {tuple(rel_logits.shape)}, expected "
|
||||
f"{(q.shape[0], q.shape[1], rel_extent)}"
|
||||
)
|
||||
|
||||
num_kv_heads = key_cache.shape[2]
|
||||
if q.shape[1] % num_kv_heads:
|
||||
raise ValueError("query heads must be divisible by KV heads")
|
||||
if out is None:
|
||||
out = torch.empty_like(q)
|
||||
gqa_group_size = q.shape[1] // num_kv_heads
|
||||
batch = cu_seqlens_q.shape[0] - 1
|
||||
if max_kv_len is None:
|
||||
max_kv_len = key_cache.shape[0] * key_cache.shape[1]
|
||||
if use_gfx950_gluon_decode(
|
||||
max_query_len=max_seqlen_q,
|
||||
page_size=key_cache.shape[1],
|
||||
head_dim=q.shape[2],
|
||||
):
|
||||
# Import only after the architecture guard. The implementation uses
|
||||
# gfx950-only CDNA4 Gluon layouts and async-copy operations.
|
||||
from vllm.models.inkling.amd.ops.gluon.rel_mha_decode_gfx950 import (
|
||||
gluon_rel_mha_decode_gfx950,
|
||||
)
|
||||
|
||||
return gluon_rel_mha_decode_gfx950(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table,
|
||||
cache_seqlens,
|
||||
max_kv_len,
|
||||
rel_logits,
|
||||
cu_seqlens_q,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
window_left=window_size[0],
|
||||
softmax_scale=softmax_scale,
|
||||
out=out,
|
||||
)
|
||||
if use_gfx950_gluon_extend(
|
||||
max_query_len=max_seqlen_q,
|
||||
max_kv_len=max_kv_len,
|
||||
page_size=key_cache.shape[1],
|
||||
head_dim=q.shape[2],
|
||||
window_left=window_size[0],
|
||||
):
|
||||
from vllm.models.inkling.amd.ops.gluon.rel_mha_extend_gfx950 import (
|
||||
gluon_rel_mha_extend_gfx950,
|
||||
)
|
||||
|
||||
# The copied kernel retains TokenSpeed's cu_seqlens_kv parameter for
|
||||
# API compatibility but does not consume it.
|
||||
return cast(
|
||||
torch.Tensor,
|
||||
gluon_rel_mha_extend_gfx950(
|
||||
q,
|
||||
cu_seqlens_q,
|
||||
cu_seqlens_q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table,
|
||||
cache_seqlens,
|
||||
window_left=window_size[0],
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
max_seqlen_k=max_kv_len,
|
||||
rel_logits=rel_logits,
|
||||
softmax_scale=softmax_scale,
|
||||
out=out,
|
||||
),
|
||||
)
|
||||
if use_split_kv_decode(
|
||||
max_query_len=max_seqlen_q,
|
||||
max_kv_len=max_kv_len,
|
||||
page_size=key_cache.shape[1],
|
||||
window_left=window_size[0],
|
||||
):
|
||||
return inkling_rel_attention_split_kv_decode(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
softmax_scale=softmax_scale,
|
||||
window_left=window_size[0],
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
max_kv_len=max_kv_len,
|
||||
out=out,
|
||||
)
|
||||
block_q = 1 if max_seqlen_q == 1 else 4
|
||||
grid = (triton.cdiv(max_seqlen_q, block_q), num_kv_heads, batch)
|
||||
_inkling_rel_attention_kernel[grid](
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
out,
|
||||
block_table,
|
||||
cache_seqlens,
|
||||
cu_seqlens_q,
|
||||
gqa_group_size,
|
||||
q.shape[2],
|
||||
softmax_scale,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
key_cache.stride(0),
|
||||
key_cache.stride(1),
|
||||
key_cache.stride(2),
|
||||
key_cache.stride(3),
|
||||
value_cache.stride(0),
|
||||
value_cache.stride(1),
|
||||
value_cache.stride(2),
|
||||
value_cache.stride(3),
|
||||
rel_logits.stride(0),
|
||||
rel_logits.stride(1),
|
||||
rel_logits.stride(2),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
out.stride(2),
|
||||
block_table.stride(0),
|
||||
page_size=key_cache.shape[1],
|
||||
rel_extent=rel_extent,
|
||||
window_left=window_size[0],
|
||||
BLOCK_Q=block_q,
|
||||
BLOCK_K=64,
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compatibility shim for the ROCm relative-attention implementation.
|
||||
|
||||
ROCm uses a Triton kernel which is compiled by vLLM's normal model warmup. The
|
||||
NVIDIA path registers ahead-of-time CuTeDSL units; importing that provider on
|
||||
ROCm would pull in CUDA-only tml-fa4 code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InklingFA4WarmupConfig:
|
||||
num_heads: int
|
||||
num_kv_heads: int
|
||||
head_dim: int
|
||||
rel_extent: int
|
||||
window_size: tuple[int, int]
|
||||
is_local: bool
|
||||
max_kv_len: int
|
||||
dtype: torch.dtype
|
||||
kv_dtype: torch.dtype
|
||||
block_size: int
|
||||
max_num_reqs: int
|
||||
max_num_batched_tokens: int
|
||||
|
||||
|
||||
def register_fa4_warmup(config: InklingFA4WarmupConfig) -> None:
|
||||
"""Triton compilation is triggered by the ordinary vLLM warmup forward."""
|
||||
del config
|
||||
@@ -0,0 +1,4 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""GFX950-only Gluon relative-attention kernels for Inkling."""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,760 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""rel_mha extend Gluon kernel for AMD GFX950.
|
||||
|
||||
This handles ragged, multi-token queries against a paged KV cache. The query
|
||||
axis is tiled into the MFMA ``M`` dimension (prefill-style): ``BLOCK_M`` query
|
||||
rows of a single q-head share each paged KV tile, so every KV tile is loaded
|
||||
once and reused across all rows in the tile. The grid is
|
||||
``(blocks_per_req, batch, n_heads)`` -- all host-known sizes -- and each program
|
||||
self-locates its request from ``cu_seqlens_q`` / ``cache_seqlens`` in-kernel, so
|
||||
the launch stays CUDA-graph static with no device->host sync.
|
||||
|
||||
Visibility per query row depends on ``is_causal`` and the optional sliding
|
||||
window:
|
||||
|
||||
* ``is_causal=False``: every query token attends the full visible cache, i.e.
|
||||
``visible_kv = cache_seqlens[batch]``.
|
||||
* ``is_causal=True``: query tokens are a causal suffix, so the ``i``-th query
|
||||
token of a request (0-indexed) attends ``prefix + i + 1`` tokens, where
|
||||
``prefix = cache_seqlens[batch] - query_len[batch]``.
|
||||
|
||||
Causal masking only touches the few KV tiles that reach the diagonal; the long
|
||||
prefix is a mask-free fast path. Sinks and sliding windows (causal or not) are
|
||||
applied per tile. This is the sole Gluon extend implementation.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.inkling.amd.ops.gluon.utils import (
|
||||
_INV_LN2_VALUE,
|
||||
_LN2,
|
||||
InputStrides,
|
||||
PagedKVStrides,
|
||||
max,
|
||||
maximum,
|
||||
)
|
||||
from vllm.triton_utils import aggregate, gl, gluon
|
||||
|
||||
cdna4 = gl.amd.cdna4
|
||||
async_copy = cdna4.async_copy
|
||||
|
||||
|
||||
# ===-----------------------------------------------------------------------===#
|
||||
# Query-batched path
|
||||
#
|
||||
# This path tiles BLOCK_M query rows (of a single q-head) into the MFMA M
|
||||
# dimension -- like the prefill kernel -- so each paged KV tile is loaded once
|
||||
# and reused across all BLOCK_M rows. KV still comes from the paged cache
|
||||
# (decode-style async page loads). Causal masking only touches the few KV tiles
|
||||
# that reach the diagonal; the long prefix is a mask-free fast path.
|
||||
# ===-----------------------------------------------------------------------===#
|
||||
|
||||
_EXTEND_SHORT_Q_BLOCK_M = 64
|
||||
_EXTEND_SHORT_Q_NUM_WARPS = 2
|
||||
_EXTEND_LONG_Q_BLOCK_M = 128
|
||||
_EXTEND_LONG_Q_NUM_WARPS = 4
|
||||
|
||||
|
||||
def _select_extend_tile(max_seqlen_q: int, page_size: int) -> tuple[int, int, int]:
|
||||
"""Return (BLOCK_M, BLOCK_N, NUM_WARPS) for the given max query length.
|
||||
|
||||
Queries that fit in a single short tile use it (least padding, most
|
||||
occupancy); longer ones use the tall tile that covers more rows per
|
||||
shared-KV pass.
|
||||
"""
|
||||
if max_seqlen_q <= _EXTEND_SHORT_Q_BLOCK_M:
|
||||
return _EXTEND_SHORT_Q_BLOCK_M, page_size, _EXTEND_SHORT_Q_NUM_WARPS
|
||||
return _EXTEND_LONG_Q_BLOCK_M, page_size, _EXTEND_LONG_Q_NUM_WARPS
|
||||
|
||||
|
||||
@aggregate
|
||||
class ExtendConfig:
|
||||
N_HEADS: gl.constexpr
|
||||
N_KV_HEADS: gl.constexpr
|
||||
GROUP_SIZE: gl.constexpr
|
||||
HEAD_DIM: gl.constexpr
|
||||
SM_SCALE: gl.constexpr
|
||||
BLOCK_M: gl.constexpr
|
||||
BLOCK_N: gl.constexpr
|
||||
NUM_WARPS: gl.constexpr
|
||||
PAGE_SIZE: gl.constexpr
|
||||
PAGE_TABLE_STRIDE: gl.constexpr
|
||||
IS_CAUSAL: gl.constexpr
|
||||
HAS_LSE: gl.constexpr
|
||||
WINDOW_LEFT: gl.constexpr
|
||||
REL_EXTENT: gl.constexpr
|
||||
REL_BIAS_QK_SCALE: gl.constexpr
|
||||
IS_FP8: gl.constexpr
|
||||
q_strides: InputStrides
|
||||
rel_strides: InputStrides
|
||||
k_strides: PagedKVStrides
|
||||
v_strides: PagedKVStrides
|
||||
qk_layout: gl.constexpr
|
||||
pv_layout: gl.constexpr
|
||||
q_layout: gl.constexpr
|
||||
k_layout: gl.constexpr
|
||||
p_layout: gl.constexpr
|
||||
v_layout: gl.constexpr
|
||||
load_layout: gl.constexpr
|
||||
store_layout: gl.constexpr
|
||||
k_smem_layout: gl.constexpr
|
||||
v_smem_layout: gl.constexpr
|
||||
|
||||
@gluon.constexpr_function
|
||||
def __init__(
|
||||
self,
|
||||
N_HEADS,
|
||||
N_KV_HEADS,
|
||||
HEAD_DIM,
|
||||
SM_SCALE,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
NUM_WARPS,
|
||||
PAGE_SIZE,
|
||||
PAGE_TABLE_STRIDE,
|
||||
IS_CAUSAL,
|
||||
HAS_LSE,
|
||||
WINDOW_LEFT,
|
||||
REL_EXTENT,
|
||||
REL_BIAS_QK_SCALE,
|
||||
IS_FP8,
|
||||
q_strides,
|
||||
rel_strides,
|
||||
k_strides,
|
||||
v_strides,
|
||||
):
|
||||
assert HEAD_DIM in (64, 128)
|
||||
assert BLOCK_N == PAGE_SIZE
|
||||
|
||||
instr_shape = [32, 32, 16]
|
||||
mfma_layout = gl.amd.AMDMFMALayout(
|
||||
version=4,
|
||||
instr_shape=instr_shape,
|
||||
transposed=True,
|
||||
warps_per_cta=[NUM_WARPS, 1],
|
||||
)
|
||||
qk_layout = mfma_layout
|
||||
pv_layout = mfma_layout
|
||||
# Elements loaded per lane depend on the input dtype, matching qk_kw.
|
||||
# load_threads is how many lanes span HEAD_DIM.
|
||||
load_vec = 16 if IS_FP8 else 8
|
||||
load_threads = HEAD_DIM // load_vec
|
||||
load_layout = gl.BlockedLayout(
|
||||
[1, load_vec], [64 // load_threads, load_threads], [NUM_WARPS, 1], [1, 0]
|
||||
)
|
||||
# Output is always 16-bit, so a 128-bit store has 8 elements.
|
||||
# store_threads is how many lanes span HEAD_DIM.
|
||||
store_vec = 8
|
||||
store_threads = HEAD_DIM // store_vec
|
||||
store_layout = gl.BlockedLayout(
|
||||
[1, store_vec], [64 // store_threads, store_threads], [NUM_WARPS, 1], [1, 0]
|
||||
)
|
||||
# Padding interval is 64 lanes * load_vec elems.
|
||||
pad_interval = 64 * load_vec
|
||||
# Empirically tuned.
|
||||
pad_k = 16 if IS_FP8 else 8
|
||||
pad_v = 16 if IS_FP8 else 32
|
||||
k_smem_layout = gl.PaddedSharedLayout.with_identity_for(
|
||||
[[pad_interval, pad_k]], [BLOCK_N, HEAD_DIM], [1, 0]
|
||||
)
|
||||
v_smem_layout = gl.PaddedSharedLayout.with_identity_for(
|
||||
[[pad_interval, pad_v]], [BLOCK_N, HEAD_DIM], [1, 0]
|
||||
)
|
||||
|
||||
self.N_HEADS = gl.constexpr(N_HEADS)
|
||||
self.N_KV_HEADS = gl.constexpr(N_KV_HEADS)
|
||||
self.GROUP_SIZE = gl.constexpr(N_HEADS // N_KV_HEADS)
|
||||
self.HEAD_DIM = gl.constexpr(HEAD_DIM)
|
||||
self.SM_SCALE = gl.constexpr(SM_SCALE)
|
||||
self.BLOCK_M = gl.constexpr(BLOCK_M)
|
||||
self.BLOCK_N = gl.constexpr(BLOCK_N)
|
||||
self.NUM_WARPS = gl.constexpr(NUM_WARPS)
|
||||
self.PAGE_SIZE = gl.constexpr(PAGE_SIZE)
|
||||
self.PAGE_TABLE_STRIDE = gl.constexpr(PAGE_TABLE_STRIDE)
|
||||
self.IS_CAUSAL = gl.constexpr(IS_CAUSAL)
|
||||
self.HAS_LSE = gl.constexpr(HAS_LSE)
|
||||
self.WINDOW_LEFT = gl.constexpr(WINDOW_LEFT)
|
||||
self.REL_EXTENT = gl.constexpr(REL_EXTENT)
|
||||
self.REL_BIAS_QK_SCALE = gl.constexpr(REL_BIAS_QK_SCALE)
|
||||
self.IS_FP8 = gl.constexpr(IS_FP8)
|
||||
self.q_strides = q_strides
|
||||
self.rel_strides = rel_strides
|
||||
self.k_strides = k_strides
|
||||
self.v_strides = v_strides
|
||||
self.qk_layout = gl.constexpr(qk_layout)
|
||||
self.pv_layout = gl.constexpr(pv_layout)
|
||||
# qk_kw is derived from a 128-bit load / dtype bitwidth.
|
||||
# pv_kw is empirically tuned.
|
||||
qk_kw = 16 if IS_FP8 else 8
|
||||
pv_kw = 16 if IS_FP8 else 4
|
||||
self.q_layout = gl.constexpr(gl.DotOperandLayout(0, qk_layout, k_width=qk_kw))
|
||||
self.k_layout = gl.constexpr(gl.DotOperandLayout(1, qk_layout, k_width=qk_kw))
|
||||
self.p_layout = gl.constexpr(gl.DotOperandLayout(0, pv_layout, k_width=pv_kw))
|
||||
self.v_layout = gl.constexpr(gl.DotOperandLayout(1, pv_layout, k_width=pv_kw))
|
||||
self.load_layout = gl.constexpr(load_layout)
|
||||
self.store_layout = gl.constexpr(store_layout)
|
||||
self.k_smem_layout = gl.constexpr(k_smem_layout)
|
||||
self.v_smem_layout = gl.constexpr(v_smem_layout)
|
||||
|
||||
|
||||
@aggregate
|
||||
class ExtendProgram:
|
||||
cfg: gl.constexpr
|
||||
q_ptr: gl.tensor
|
||||
rel_logits_ptr: gl.tensor
|
||||
k_cache_ptr: gl.tensor
|
||||
v_cache_ptr: gl.tensor
|
||||
page_table_ptr: gl.tensor
|
||||
output_ptr: gl.tensor
|
||||
lse_ptr: gl.tensor
|
||||
batch: gl.tensor
|
||||
q_head: gl.tensor
|
||||
kv_head: gl.tensor
|
||||
q_start: gl.tensor
|
||||
seq_base: gl.tensor
|
||||
seq_len: gl.tensor
|
||||
prefix: gl.tensor
|
||||
cache_len: gl.tensor
|
||||
|
||||
@gluon.constexpr_function
|
||||
def __init__(
|
||||
self,
|
||||
cfg,
|
||||
q_ptr,
|
||||
rel_logits_ptr,
|
||||
k_cache_ptr,
|
||||
v_cache_ptr,
|
||||
page_table_ptr,
|
||||
output_ptr,
|
||||
lse_ptr,
|
||||
batch,
|
||||
q_head,
|
||||
kv_head,
|
||||
q_start,
|
||||
seq_base,
|
||||
seq_len,
|
||||
prefix,
|
||||
cache_len,
|
||||
):
|
||||
self.cfg = gl.constexpr(cfg)
|
||||
self.q_ptr = q_ptr
|
||||
self.rel_logits_ptr = rel_logits_ptr
|
||||
self.k_cache_ptr = k_cache_ptr
|
||||
self.v_cache_ptr = v_cache_ptr
|
||||
self.page_table_ptr = page_table_ptr
|
||||
self.output_ptr = output_ptr
|
||||
self.lse_ptr = lse_ptr
|
||||
self.batch = batch
|
||||
self.q_head = q_head
|
||||
self.kv_head = kv_head
|
||||
self.q_start = q_start
|
||||
self.seq_base = seq_base
|
||||
self.seq_len = seq_len
|
||||
self.prefix = prefix
|
||||
self.cache_len = cache_len
|
||||
|
||||
@gluon.jit
|
||||
def create(
|
||||
cfg,
|
||||
q_ptr,
|
||||
rel_logits_ptr,
|
||||
k_cache_ptr,
|
||||
v_cache_ptr,
|
||||
page_table_ptr,
|
||||
output_ptr,
|
||||
lse_ptr,
|
||||
cu_seqlens_q_ptr,
|
||||
cache_seqlens_ptr,
|
||||
):
|
||||
# Gluon treats ``cfg`` as the constexpr factory argument, while mypy
|
||||
# models the first parameter of this aggregate method as ``self``.
|
||||
block_in_req = gl.program_id(0)
|
||||
batch = gl.program_id(1)
|
||||
q_head = gl.program_id(2)
|
||||
q_start = block_in_req * cfg.BLOCK_M # type: ignore[attr-defined]
|
||||
seq_base = gl.load(cu_seqlens_q_ptr + batch)
|
||||
seq_end = gl.load(cu_seqlens_q_ptr + batch + 1)
|
||||
seq_len = seq_end - seq_base
|
||||
cache_len = gl.load(cache_seqlens_ptr + batch)
|
||||
prefix = cache_len - seq_len
|
||||
kv_head = q_head // cfg.GROUP_SIZE # type: ignore[attr-defined]
|
||||
return ExtendProgram(
|
||||
gl.constexpr(cfg),
|
||||
q_ptr,
|
||||
rel_logits_ptr,
|
||||
k_cache_ptr,
|
||||
v_cache_ptr,
|
||||
page_table_ptr,
|
||||
output_ptr,
|
||||
lse_ptr,
|
||||
batch,
|
||||
q_head,
|
||||
kv_head,
|
||||
q_start,
|
||||
seq_base,
|
||||
seq_len,
|
||||
prefix,
|
||||
cache_len,
|
||||
)
|
||||
|
||||
@gluon.jit
|
||||
def load_q(self):
|
||||
cfg = self.cfg
|
||||
offs_m = self.q_start + gl.arange(
|
||||
0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.q_layout)
|
||||
)
|
||||
offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.q_layout))
|
||||
row = self.seq_base + offs_m
|
||||
offsets = cfg.q_strides.offsets(row[:, None], self.q_head, offs_d[None, :])
|
||||
mask = offs_m[:, None] < self.seq_len
|
||||
return cdna4.buffer_load(self.q_ptr, offsets, mask=mask, other=0.0)
|
||||
|
||||
@gluon.jit
|
||||
def load_page(self, start_n):
|
||||
cfg = self.cfg
|
||||
page_index = start_n // cfg.PAGE_SIZE
|
||||
valid = start_n < self.cache_len
|
||||
return gl.load(
|
||||
self.page_table_ptr + self.batch * cfg.PAGE_TABLE_STRIDE + page_index,
|
||||
mask=valid,
|
||||
other=0,
|
||||
)
|
||||
|
||||
@gluon.jit
|
||||
def issue_load_k(self, physical_page, k_smem):
|
||||
cfg = self.cfg
|
||||
offs_n = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(1, cfg.load_layout))
|
||||
offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.load_layout))
|
||||
offsets = cfg.k_strides.offsets(
|
||||
physical_page,
|
||||
offs_n[:, None],
|
||||
self.kv_head,
|
||||
offs_d[None, :],
|
||||
)
|
||||
async_copy.global_load_to_shared(k_smem, self.k_cache_ptr + offsets)
|
||||
async_copy.commit_group()
|
||||
|
||||
@gluon.jit
|
||||
def issue_load_v(self, physical_page, v_smem):
|
||||
cfg = self.cfg
|
||||
offs_n = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(1, cfg.load_layout))
|
||||
offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.load_layout))
|
||||
offsets = cfg.v_strides.offsets(
|
||||
physical_page,
|
||||
offs_n[:, None],
|
||||
self.kv_head,
|
||||
offs_d[None, :],
|
||||
)
|
||||
async_copy.global_load_to_shared(v_smem, self.v_cache_ptr + offsets)
|
||||
async_copy.commit_group()
|
||||
|
||||
@gluon.jit
|
||||
def shared_load_k(self, k_smem):
|
||||
cfg = self.cfg
|
||||
k_buffer = k_smem.permute([1, 0])
|
||||
return k_buffer.load(cfg.k_layout)
|
||||
|
||||
@gluon.jit
|
||||
def shared_load_v(self, v_smem):
|
||||
cfg = self.cfg
|
||||
return v_smem.load(cfg.v_layout)
|
||||
|
||||
@gluon.jit
|
||||
def compute_qk(self, q, k):
|
||||
cfg = self.cfg
|
||||
qk = gl.zeros(
|
||||
[cfg.BLOCK_M, cfg.BLOCK_N], dtype=gl.float32, layout=cfg.qk_layout
|
||||
)
|
||||
return cdna4.mfma(q, k, qk)
|
||||
|
||||
@gluon.jit
|
||||
def apply_rel_bias(self, qk, start_n):
|
||||
cfg = self.cfg
|
||||
offs_m = self.q_start + gl.arange(
|
||||
0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.qk_layout)
|
||||
)
|
||||
offs_n_abs = start_n + gl.arange(
|
||||
0, cfg.BLOCK_N, layout=gl.SliceLayout(0, cfg.qk_layout)
|
||||
)
|
||||
q_pos = self.prefix + offs_m
|
||||
rel_dist = q_pos[:, None] - offs_n_abs[None, :]
|
||||
rel_valid = (rel_dist >= 0) & (rel_dist < cfg.REL_EXTENT)
|
||||
rel_idx = gl.where(rel_dist >= 0, rel_dist, 0)
|
||||
rel_idx = gl.where(rel_idx < cfg.REL_EXTENT, rel_idx, cfg.REL_EXTENT - 1)
|
||||
offsets = cfg.rel_strides.offsets(
|
||||
self.seq_base + offs_m[:, None], self.q_head, rel_idx
|
||||
)
|
||||
mask = (offs_m[:, None] < self.seq_len) & rel_valid
|
||||
rel_bias = cdna4.buffer_load(
|
||||
self.rel_logits_ptr, offsets, mask=mask, other=0.0
|
||||
).to(gl.float32)
|
||||
return qk + rel_bias * cfg.REL_BIAS_QK_SCALE
|
||||
|
||||
@gluon.jit
|
||||
def compute_pv(self, p, v, acc):
|
||||
return cdna4.mfma(p, v, acc)
|
||||
|
||||
@gluon.jit
|
||||
def init_attention_state(self):
|
||||
cfg = self.cfg
|
||||
m_i = gl.full(
|
||||
[cfg.BLOCK_M],
|
||||
value=-float("inf"),
|
||||
dtype=gl.float32,
|
||||
layout=gl.SliceLayout(1, cfg.pv_layout),
|
||||
)
|
||||
l_i = gl.full(
|
||||
[cfg.BLOCK_M],
|
||||
value=0,
|
||||
dtype=gl.float32,
|
||||
layout=gl.SliceLayout(1, cfg.pv_layout),
|
||||
)
|
||||
acc = gl.zeros(
|
||||
[cfg.BLOCK_M, cfg.HEAD_DIM], dtype=gl.float32, layout=cfg.pv_layout
|
||||
)
|
||||
return m_i, l_i, acc
|
||||
|
||||
@gluon.jit
|
||||
def softmax(self, qk, m_i, l_i, acc):
|
||||
cfg = self.cfg
|
||||
# In sliding window case, some rows can see fully masked tiles before
|
||||
# any valid KV. Guard the online softmax state so `-inf - -inf` does not
|
||||
# produce NaNs.
|
||||
HAS_INVALID: gl.constexpr = cfg.WINDOW_LEFT >= 0
|
||||
|
||||
row_max = max(qk, 1)
|
||||
m_new = maximum(m_i, row_max)
|
||||
m_new_scaled = m_new * cfg.SM_SCALE
|
||||
if HAS_INVALID:
|
||||
invalid = m_new == -float("inf")
|
||||
m_new_scaled = gl.where(invalid, 0.0, m_new_scaled)
|
||||
|
||||
qk_shifted = qk * cfg.SM_SCALE - m_new_scaled[:, None]
|
||||
p = gl.exp2(qk_shifted)
|
||||
m_diff = m_i * cfg.SM_SCALE - m_new_scaled
|
||||
if HAS_INVALID:
|
||||
m_diff = gl.where(invalid, 0.0, m_diff)
|
||||
|
||||
alpha = gl.exp2(m_diff)
|
||||
l_ij = gl.sum(p, axis=1)
|
||||
l_i = l_i * alpha + l_ij
|
||||
acc = acc * alpha[:, None]
|
||||
p = p.to(self.q_ptr.dtype.element_ty)
|
||||
p = gl.convert_layout(p, cfg.p_layout)
|
||||
return p, m_new, l_i, acc
|
||||
|
||||
@gluon.jit
|
||||
def store_output(self, output):
|
||||
cfg = self.cfg
|
||||
offs_m = self.q_start + gl.arange(
|
||||
0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.store_layout)
|
||||
)
|
||||
offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.store_layout))
|
||||
offsets = (
|
||||
((self.seq_base + offs_m[:, None]) * cfg.N_HEADS + self.q_head)
|
||||
* cfg.HEAD_DIM
|
||||
+ offs_d[None, :]
|
||||
).to(gl.int32)
|
||||
mask = offs_m[:, None] < self.seq_len
|
||||
output = output.to(self.output_ptr.dtype.element_ty)
|
||||
cdna4.buffer_store(output, self.output_ptr, offsets, mask=mask)
|
||||
|
||||
@gluon.jit
|
||||
def store_lse(self, l_i, m_i):
|
||||
cfg = self.cfg
|
||||
if cfg.HAS_LSE:
|
||||
offs_m = self.q_start + gl.arange(
|
||||
0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.pv_layout)
|
||||
)
|
||||
offsets = ((self.seq_base + offs_m) * cfg.N_HEADS + self.q_head).to(
|
||||
gl.int32
|
||||
)
|
||||
mask = offs_m < self.seq_len
|
||||
lse_l_i = gl.where(l_i > 0.0, l_i, 1.0)
|
||||
# Softmax runs in base-2 (exp2 hardware fast path), so m_i*SM_SCALE +
|
||||
# log2(l_i) is the LSE in base-2 units. Convert to natural log (the
|
||||
# public op contract / torch.logsumexp convention) by scaling by ln2.
|
||||
lse = (m_i * cfg.SM_SCALE + gl.log2(lse_l_i)) * _LN2
|
||||
cdna4.buffer_store(lse, self.lse_ptr, offsets, mask=mask)
|
||||
|
||||
|
||||
@gluon.jit
|
||||
def _rel_mha_extend_fp16(
|
||||
q_ptr,
|
||||
rel_logits_ptr,
|
||||
k_cache_ptr,
|
||||
v_cache_ptr,
|
||||
page_table_ptr,
|
||||
output_ptr,
|
||||
lse_ptr,
|
||||
cu_seqlens_q_ptr,
|
||||
cache_seqlens_ptr,
|
||||
Q_STRIDE_T: gl.constexpr,
|
||||
Q_STRIDE_H: gl.constexpr,
|
||||
Q_STRIDE_D: gl.constexpr,
|
||||
K_STRIDE_B: gl.constexpr,
|
||||
K_STRIDE_P: gl.constexpr,
|
||||
K_STRIDE_H: gl.constexpr,
|
||||
K_STRIDE_D: gl.constexpr,
|
||||
V_STRIDE_B: gl.constexpr,
|
||||
V_STRIDE_P: gl.constexpr,
|
||||
V_STRIDE_H: gl.constexpr,
|
||||
V_STRIDE_D: gl.constexpr,
|
||||
SM_SCALE: gl.constexpr,
|
||||
PAGE_TABLE_STRIDE: gl.constexpr,
|
||||
PAGE_SIZE: gl.constexpr,
|
||||
N_HEADS: gl.constexpr,
|
||||
N_KV_HEADS: gl.constexpr,
|
||||
HEAD_DIM: gl.constexpr,
|
||||
BLOCK_M: gl.constexpr,
|
||||
BLOCK_N: gl.constexpr,
|
||||
NUM_WARPS: gl.constexpr,
|
||||
IS_CAUSAL: gl.constexpr,
|
||||
HAS_LSE: gl.constexpr,
|
||||
WINDOW_LEFT: gl.constexpr,
|
||||
REL_STRIDE_T: gl.constexpr,
|
||||
REL_STRIDE_H: gl.constexpr,
|
||||
REL_STRIDE_E: gl.constexpr,
|
||||
REL_EXTENT: gl.constexpr,
|
||||
REL_BIAS_QK_SCALE: gl.constexpr,
|
||||
IS_FP8: gl.constexpr,
|
||||
):
|
||||
cfg = ExtendConfig(
|
||||
N_HEADS,
|
||||
N_KV_HEADS,
|
||||
HEAD_DIM,
|
||||
SM_SCALE,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
NUM_WARPS,
|
||||
PAGE_SIZE,
|
||||
PAGE_TABLE_STRIDE,
|
||||
IS_CAUSAL,
|
||||
HAS_LSE,
|
||||
WINDOW_LEFT,
|
||||
REL_EXTENT,
|
||||
REL_BIAS_QK_SCALE,
|
||||
IS_FP8,
|
||||
InputStrides(Q_STRIDE_T, Q_STRIDE_H, Q_STRIDE_D),
|
||||
InputStrides(REL_STRIDE_T, REL_STRIDE_H, REL_STRIDE_E),
|
||||
PagedKVStrides(K_STRIDE_B, K_STRIDE_P, K_STRIDE_H, K_STRIDE_D),
|
||||
PagedKVStrides(V_STRIDE_B, V_STRIDE_P, V_STRIDE_H, V_STRIDE_D),
|
||||
)
|
||||
program = ExtendProgram.create(
|
||||
cfg,
|
||||
q_ptr,
|
||||
rel_logits_ptr,
|
||||
k_cache_ptr,
|
||||
v_cache_ptr,
|
||||
page_table_ptr,
|
||||
output_ptr,
|
||||
lse_ptr,
|
||||
cu_seqlens_q_ptr,
|
||||
cache_seqlens_ptr,
|
||||
)
|
||||
# Over-provisioned tile past this request's real query rows: nothing to do.
|
||||
if program.q_start >= program.seq_len:
|
||||
return
|
||||
k_smem = gl.allocate_shared_memory(
|
||||
k_cache_ptr.dtype.element_ty, [cfg.BLOCK_N, cfg.HEAD_DIM], cfg.k_smem_layout
|
||||
)
|
||||
v_smem = gl.allocate_shared_memory(
|
||||
v_cache_ptr.dtype.element_ty, [cfg.BLOCK_N, cfg.HEAD_DIM], cfg.v_smem_layout
|
||||
)
|
||||
|
||||
q = program.load_q()
|
||||
m_i, l_i, acc = program.init_attention_state()
|
||||
|
||||
offs_m_q = program.q_start + gl.arange(
|
||||
0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.qk_layout)
|
||||
)
|
||||
offs_n_q = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(0, cfg.qk_layout))
|
||||
diag_row = program.prefix + offs_m_q
|
||||
|
||||
if IS_CAUSAL:
|
||||
kv_end = min(program.cache_len, program.prefix + program.q_start + cfg.BLOCK_M)
|
||||
else:
|
||||
kv_end = program.cache_len
|
||||
|
||||
# Sliding window (inclusive-left, matches flash-attn window_size=(W, 0)):
|
||||
# skip KV tiles entirely below the window's lower edge. The tile's top query
|
||||
# row sits at absolute position pos_top; its window opens at
|
||||
# pos_top - WINDOW_LEFT (that key is visible -> W + 1 keys total). The min()
|
||||
# form clamps to 0 without the shadowed builtin max().
|
||||
if cfg.WINDOW_LEFT >= 0:
|
||||
pos_top = program.prefix + program.q_start
|
||||
kv_start = pos_top - min(pos_top, cfg.WINDOW_LEFT)
|
||||
kv_start = (kv_start // cfg.BLOCK_N) * cfg.BLOCK_N
|
||||
else:
|
||||
kv_start = 0
|
||||
|
||||
for start_n in range(kv_start, kv_end, cfg.BLOCK_N):
|
||||
physical_page = program.load_page(start_n)
|
||||
program.issue_load_k(physical_page, k_smem)
|
||||
program.issue_load_v(physical_page, v_smem)
|
||||
|
||||
async_copy.wait_group(1)
|
||||
k = program.shared_load_k(k_smem)
|
||||
qk = program.compute_qk(q, k)
|
||||
qk = program.apply_rel_bias(qk, start_n)
|
||||
|
||||
if cfg.WINDOW_LEFT >= 0:
|
||||
# Window lower edge + cache bound always apply; causal upper edge only
|
||||
# when IS_CAUSAL (independent layering keeps non-causal + window correct).
|
||||
offs_n_abs = start_n + offs_n_q
|
||||
mask = (offs_n_abs[None, :] >= diag_row[:, None] - cfg.WINDOW_LEFT) & (
|
||||
offs_n_abs[None, :] < program.cache_len
|
||||
)
|
||||
if IS_CAUSAL:
|
||||
mask &= offs_n_abs[None, :] <= diag_row[:, None]
|
||||
qk = gl.where(mask, qk, -float("inf"))
|
||||
elif IS_CAUSAL:
|
||||
if start_n + cfg.BLOCK_N > program.prefix + program.q_start:
|
||||
offs_n_abs = start_n + offs_n_q
|
||||
mask = (offs_n_abs[None, :] <= diag_row[:, None]) & (
|
||||
offs_n_abs[None, :] < program.cache_len
|
||||
)
|
||||
qk = gl.where(mask, qk, -float("inf"))
|
||||
else:
|
||||
if start_n + cfg.BLOCK_N > program.cache_len:
|
||||
offs_n_abs = start_n + offs_n_q
|
||||
qk = gl.where(
|
||||
offs_n_abs[None, :] < program.cache_len, qk, -float("inf")
|
||||
)
|
||||
|
||||
p, m_i, l_i, acc = program.softmax(qk, m_i, l_i, acc)
|
||||
|
||||
async_copy.wait_group(0)
|
||||
v = program.shared_load_v(v_smem)
|
||||
acc = program.compute_pv(p, v, acc)
|
||||
|
||||
denom = gl.where(l_i > 0.0, l_i, 1.0)
|
||||
output = acc * (1.0 / denom)[:, None]
|
||||
output = gl.convert_layout(output, cfg.store_layout)
|
||||
program.store_output(output)
|
||||
program.store_lse(l_i, m_i)
|
||||
|
||||
|
||||
def gluon_rel_mha_extend_gfx950(
|
||||
q: torch.Tensor,
|
||||
cu_seqlens_q: torch.Tensor,
|
||||
cu_seqlens_kv: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
cache_seqlens: torch.Tensor,
|
||||
window_left: int = -1,
|
||||
return_lse: bool = False,
|
||||
max_seqlen_q: int = 1,
|
||||
max_seqlen_k: int = 1,
|
||||
rel_logits: torch.Tensor | None = None,
|
||||
softmax_scale: float | None = None,
|
||||
q_scale: torch.Tensor | None = None,
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
assert rel_logits is not None
|
||||
head_dim = q.shape[2]
|
||||
n_heads = q.shape[1]
|
||||
n_kv_heads = k_cache.shape[2]
|
||||
page_size = k_cache.shape[1]
|
||||
block_m, block_n, num_warps = _select_extend_tile(max_seqlen_q, page_size)
|
||||
if softmax_scale is None:
|
||||
softmax_scale = 1.0 / math.sqrt(head_dim)
|
||||
sm_scale = softmax_scale * _INV_LN2_VALUE
|
||||
rel_bias_qk_scale = 1.0 / softmax_scale
|
||||
|
||||
# max_seqlen_q must be >= the true max query length; extra tiles early-exit.
|
||||
batch = cu_seqlens_q.shape[0] - 1
|
||||
safe_max_q = max_seqlen_q if max_seqlen_q > 0 else 1
|
||||
blocks_per_req = (safe_max_q + block_m - 1) // block_m
|
||||
cu_q_i32 = cu_seqlens_q.to(torch.int32).contiguous()
|
||||
cache_i32 = cache_seqlens.to(torch.int32).contiguous()
|
||||
|
||||
is_fp8 = q.dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
|
||||
out_dtype = torch.bfloat16 if is_fp8 else q.dtype
|
||||
if page_size not in (64, 128, 256):
|
||||
raise ValueError(
|
||||
"gfx950 Gluon relative-attention extend requires page size "
|
||||
f"64, 128, or 256; got {page_size}"
|
||||
)
|
||||
output = (
|
||||
torch.empty(q.shape, device=q.device, dtype=out_dtype) if out is None else out
|
||||
)
|
||||
if return_lse:
|
||||
lse = torch.empty((q.shape[0], n_heads), device=q.device, dtype=torch.float32)
|
||||
lse_arg = lse
|
||||
else:
|
||||
lse = None
|
||||
lse_arg = q
|
||||
grid = (blocks_per_req, batch, n_heads)
|
||||
_rel_mha_extend_fp16[grid](
|
||||
q,
|
||||
rel_logits,
|
||||
k_cache,
|
||||
v_cache,
|
||||
page_table,
|
||||
output,
|
||||
lse_arg,
|
||||
cu_q_i32,
|
||||
cache_i32,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
k_cache.stride(0),
|
||||
k_cache.stride(1),
|
||||
k_cache.stride(2),
|
||||
k_cache.stride(3),
|
||||
v_cache.stride(0),
|
||||
v_cache.stride(1),
|
||||
v_cache.stride(2),
|
||||
v_cache.stride(3),
|
||||
sm_scale,
|
||||
page_table.stride(0),
|
||||
page_size,
|
||||
n_heads,
|
||||
n_kv_heads,
|
||||
head_dim,
|
||||
block_m,
|
||||
block_n,
|
||||
num_warps,
|
||||
True,
|
||||
return_lse,
|
||||
window_left,
|
||||
rel_logits.stride(0),
|
||||
rel_logits.stride(1),
|
||||
rel_logits.stride(2),
|
||||
rel_logits.shape[2],
|
||||
rel_bias_qk_scale,
|
||||
is_fp8,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
if return_lse:
|
||||
assert lse is not None
|
||||
return output, lse
|
||||
return output
|
||||
@@ -0,0 +1,154 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from vllm.triton_utils import aggregate, gl, gluon, tl
|
||||
|
||||
_INV_LN2_VALUE = 1.4426950408889634
|
||||
_INV_LN2 = tl.constexpr(_INV_LN2_VALUE)
|
||||
_LN2_VALUE = 0.6931471805599453
|
||||
_LN2 = tl.constexpr(_LN2_VALUE)
|
||||
_PROPAGATE_NAN_ALL = gl.constexpr(tl.PropagateNan.ALL)
|
||||
|
||||
|
||||
@gluon.jit
|
||||
def maximum(a, b, propagate_nan: gl.constexpr = _PROPAGATE_NAN_ALL):
|
||||
return gl.maximum(a, b, propagate_nan=propagate_nan)
|
||||
|
||||
|
||||
@gluon.jit
|
||||
def max(input, axis=None, keep_dims=False):
|
||||
return gl.reduce(input, axis, maximum, keep_dims=keep_dims)
|
||||
|
||||
|
||||
@gluon.constexpr_function
|
||||
def attention_layouts(head_dim, block_n, is_fp8, dtype, num_warps, instr_shape):
|
||||
mfma = gl.amd.AMDMFMALayout(
|
||||
version=4,
|
||||
instr_shape=instr_shape,
|
||||
transposed=True,
|
||||
warps_per_cta=[num_warps, 1],
|
||||
)
|
||||
qk_layout = mfma
|
||||
pv_layout = mfma
|
||||
# qk_kw is derived from a 128-bit load / dtype bitwidth; pv_kw is tuned.
|
||||
qk_kw = 16 if is_fp8 else 8
|
||||
pv_kw = 8 if is_fp8 else 4
|
||||
q_layout = gl.DotOperandLayout(0, qk_layout, k_width=qk_kw)
|
||||
k_layout = gl.DotOperandLayout(1, qk_layout, k_width=qk_kw)
|
||||
p_layout = gl.DotOperandLayout(0, pv_layout, k_width=pv_kw)
|
||||
v_layout = gl.DotOperandLayout(1, pv_layout, k_width=pv_kw)
|
||||
# load_vec = elems/lane (dtype-dependent, == qk_kw); load_threads span HEAD_DIM.
|
||||
load_vec = 16 if is_fp8 else 8
|
||||
load_threads = head_dim // load_vec
|
||||
load_layout = gl.BlockedLayout(
|
||||
[1, load_vec], [64 // load_threads, load_threads], [num_warps, 1], [1, 0]
|
||||
)
|
||||
# store_vec is always 16-bit (128 / 16 = 8) regardless of input dtype.
|
||||
store_vec = 8
|
||||
store_threads = head_dim // store_vec
|
||||
store_layout = gl.BlockedLayout(
|
||||
[1, store_vec], [64 // store_threads, store_threads], [num_warps, 1], [1, 0]
|
||||
)
|
||||
# Take only the built-in's padding, not its swizzle: the swizzle scatters
|
||||
# head_dim across banks, making the LDS write stride non-constant, so it can't
|
||||
# lower through async_copy's affine [1, 0] load. Padding-only is affine and
|
||||
# DMA-legal.
|
||||
# TODO(perf): to also use the swizzle, co-design a matched load layout so the
|
||||
# DMA stays legal.
|
||||
k_api = gl.amd.cdna4.compute_efficient_padded_shared_layout(
|
||||
k_layout, [block_n, head_dim], dtype, is_k_contig=True
|
||||
)
|
||||
v_api = gl.amd.cdna4.compute_efficient_padded_shared_layout(
|
||||
v_layout, [block_n, head_dim], dtype, is_k_contig=False
|
||||
)
|
||||
assert k_api is not None and v_api is not None, (
|
||||
"no CDNA4 padded shared layout for this operand/dtype"
|
||||
)
|
||||
k_pairs = list(k_api.interval_padding_pairs)
|
||||
v_pairs = list(v_api.interval_padding_pairs)
|
||||
assert len(k_pairs) == 1 and len(v_pairs) == 1, (
|
||||
"expected a single interval padding pair from the built-in"
|
||||
)
|
||||
k_smem_layout = gl.PaddedSharedLayout.with_identity_for(
|
||||
[[int(k_pairs[0][0]), int(k_pairs[0][1])]], [block_n, head_dim], [1, 0]
|
||||
)
|
||||
v_smem_layout = gl.PaddedSharedLayout.with_identity_for(
|
||||
[[int(v_pairs[0][0]), int(v_pairs[0][1])]], [block_n, head_dim], [1, 0]
|
||||
)
|
||||
return (
|
||||
qk_layout,
|
||||
pv_layout,
|
||||
q_layout,
|
||||
k_layout,
|
||||
p_layout,
|
||||
v_layout,
|
||||
load_layout,
|
||||
store_layout,
|
||||
k_smem_layout,
|
||||
v_smem_layout,
|
||||
)
|
||||
|
||||
|
||||
@aggregate
|
||||
class InputStrides:
|
||||
stride_t: gl.constexpr
|
||||
stride_h: gl.constexpr
|
||||
stride_d: gl.constexpr
|
||||
|
||||
@gluon.constexpr_function
|
||||
def __init__(self, stride_t, stride_h, stride_d):
|
||||
self.stride_t = gl.constexpr(stride_t)
|
||||
self.stride_h = gl.constexpr(stride_h)
|
||||
self.stride_d = gl.constexpr(stride_d)
|
||||
|
||||
@gluon.jit
|
||||
def offsets(self, token, head, dim):
|
||||
return (token * self.stride_t + head * self.stride_h + dim * self.stride_d).to(
|
||||
gl.int32
|
||||
)
|
||||
|
||||
|
||||
@aggregate
|
||||
class PagedKVStrides:
|
||||
stride_b: gl.constexpr
|
||||
stride_p: gl.constexpr
|
||||
stride_h: gl.constexpr
|
||||
stride_d: gl.constexpr
|
||||
|
||||
@gluon.constexpr_function
|
||||
def __init__(self, stride_b, stride_p, stride_h, stride_d):
|
||||
self.stride_b = gl.constexpr(stride_b)
|
||||
self.stride_p = gl.constexpr(stride_p)
|
||||
self.stride_h = gl.constexpr(stride_h)
|
||||
self.stride_d = gl.constexpr(stride_d)
|
||||
|
||||
@gluon.jit
|
||||
def offsets(self, page, token, head, dim):
|
||||
# KV pools can exceed the 32-bit buffer-offset range. Keep the page
|
||||
# arithmetic in int64, matching the original kernel's large-cache path.
|
||||
return (
|
||||
page.to(gl.int64) * self.stride_b
|
||||
+ token.to(gl.int64) * self.stride_p
|
||||
+ head * self.stride_h
|
||||
+ dim * self.stride_d
|
||||
)
|
||||
@@ -0,0 +1,766 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Deadlock-free fused RS + short-conv + AG + residual + RMSNorm.
|
||||
|
||||
The public integration surface is ``LamportRSConv.rs_sconv_ag_add_norm``.
|
||||
|
||||
Liveness
|
||||
--------
|
||||
Large grids are deliberately split at the two communication dependencies:
|
||||
|
||||
1. ``_publish_input_kernel`` only publishes rank partials.
|
||||
2. ``_reduce_insert_kernel`` waits, reduces, and inserts the local shard.
|
||||
3. ``_sconv_publish_kernel`` only computes and publishes the local result.
|
||||
4. ``_gather_norm_kernel`` waits, gathers, and normalizes.
|
||||
|
||||
Without PDL, CUDA stream order completes a producer before its consumer. With
|
||||
PDL, every producer CTA posts all peer stores before triggering its dependent,
|
||||
and the consumer executes ``gdc_wait`` before polling. A consumer therefore
|
||||
waits only for stores from a producer that is already running (or complete) on
|
||||
another GPU. It never waits for another block in its own grid. Consequently
|
||||
spinning consumers cannot occupy resources needed by any producer, and the
|
||||
wait-for graph has no cycle. The proof is independent of grid size, block
|
||||
dispatch order, and occupancy.
|
||||
|
||||
For one token, the first three phases use eight independent channel slices to
|
||||
expose enough CTA parallelism for decode latency. Each slice has exclusive
|
||||
ownership of its cache and Lamport columns; the gather/RMSNorm phase retains
|
||||
one CTA per token so no cross-CTA reduction or completion counter is needed.
|
||||
|
||||
Immediate buffer reuse (including replay of a captured CUDA graph) is also
|
||||
safe. Rank R cannot republish input for call n+1 until its gather for n has
|
||||
finished; that gather waited for owner O's output, which O publishes only
|
||||
after consuming R's input for n. Likewise, R cannot republish output for n+1
|
||||
until its reduction for n+1 has observed destination D's input; D publishes
|
||||
that input only after its gather consumed R's output for n. Thus every prior
|
||||
read happens-before a same-slot rewrite. Three generations reduce incidental
|
||||
coupling for ordinary launches, but correctness does not depend on rotation.
|
||||
|
||||
The payload itself is the Lamport flag. A 32-bit store publishes two bf16s
|
||||
atomically; 0x80008000 (two negative zeroes) denotes an empty pair. Real
|
||||
negative zeroes are changed to positive zero before publication. Consumers
|
||||
use volatile 32-bit loads and restore the sentinel after consuming a slot.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.distributed.parallel_state import in_the_same_node_as
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MAX_TOKENS = 16384
|
||||
_EMPTY_PAIR = tl.constexpr(0x80008000)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_bf16_pairs(values):
|
||||
"""Pack bf16 values into atomic u32 pairs and reserve negative zero."""
|
||||
lo, hi = tl.split(values.reshape([values.shape[0] // 2, 2]))
|
||||
lo = lo.to(tl.uint16, bitcast=True)
|
||||
hi = hi.to(tl.uint16, bitcast=True)
|
||||
lo = tl.where(lo == 0x8000, 0, lo).to(tl.uint32)
|
||||
hi = tl.where(hi == 0x8000, 0, hi).to(tl.uint32)
|
||||
return lo | (hi << 16)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _unpack_bf16_pairs(values):
|
||||
lo = (values & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True)
|
||||
hi = (values >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True)
|
||||
return tl.interleave(lo, hi)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _wait_pairs(ptr, offsets, mask):
|
||||
values = tl.load(ptr + offsets, mask=mask, other=0, volatile=True)
|
||||
while tl.max(tl.where(mask & (values == _EMPTY_PAIR), 1, 0)) != 0:
|
||||
values = tl.load(ptr + offsets, mask=mask, other=0, volatile=True)
|
||||
return values
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _publish_input_kernel(
|
||||
stage_ptr,
|
||||
peer_ptrs,
|
||||
peer_offset_u32,
|
||||
stride_stage_t,
|
||||
C: tl.constexpr,
|
||||
CS: tl.constexpr,
|
||||
CS_P2: tl.constexpr,
|
||||
SPLITS: tl.constexpr,
|
||||
RANK: tl.constexpr,
|
||||
WORLD: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""Publish this rank's full partial row into every shard owner's slots."""
|
||||
token = tl.program_id(0).to(tl.int64)
|
||||
split = tl.program_id(1).to(tl.int64)
|
||||
CSS: tl.constexpr = CS // SPLITS
|
||||
pair = tl.arange(0, CS_P2 // 2)
|
||||
pair_mask = pair < CSS // 2
|
||||
elem = tl.arange(0, CS_P2)
|
||||
elem_mask = elem < CSS
|
||||
ptrs = peer_ptrs.to(tl.pointer_type(tl.uint64))
|
||||
# Address generation is independent of the preceding stream kernel. The
|
||||
# acquire remains before stage/peer loads, which may consume its writes.
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
for owner in tl.static_range(WORLD):
|
||||
values = tl.load(
|
||||
stage_ptr + token * stride_stage_t + owner * CS + split * CSS + elem,
|
||||
mask=elem_mask,
|
||||
other=0.0,
|
||||
)
|
||||
packed = _pack_bf16_pairs(values)
|
||||
base = tl.load(ptrs + owner).to(tl.pointer_type(tl.uint32))
|
||||
dst = (token * WORLD + RANK) * (CS // 2) + split * (CSS // 2) + pair
|
||||
tl.store(base + peer_offset_u32 + dst, packed, mask=pair_mask)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _reduce_insert_kernel(
|
||||
input_peer_ptrs,
|
||||
input_peer_offset_u32,
|
||||
cache_ptr,
|
||||
slot_ptr,
|
||||
stride_cache_block,
|
||||
stride_cache_head,
|
||||
stride_cache_token,
|
||||
stride_cache_dim,
|
||||
block_size,
|
||||
C: tl.constexpr,
|
||||
CS: tl.constexpr,
|
||||
CS_P2: tl.constexpr,
|
||||
SPLITS: tl.constexpr,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
CACHE_OFFSET: tl.constexpr,
|
||||
RANK: tl.constexpr,
|
||||
WORLD: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""Consume all partials for this rank and insert the reduced cache row."""
|
||||
token = tl.program_id(0).to(tl.int64)
|
||||
split = tl.program_id(1).to(tl.int64)
|
||||
CSS: tl.constexpr = CS // SPLITS
|
||||
source = tl.arange(0, WORLD)
|
||||
pair = tl.arange(0, CS_P2 // 2)
|
||||
pair_mask = pair < CSS // 2
|
||||
offsets = (
|
||||
(token * WORLD + source)[:, None] * (CS // 2)
|
||||
+ split * (CSS // 2)
|
||||
+ pair[None, :]
|
||||
)
|
||||
mask = tl.full([WORLD], True, tl.int1)[:, None] & pair_mask[None, :]
|
||||
input_ptrs = input_peer_ptrs.to(tl.pointer_type(tl.uint64))
|
||||
input_u32 = tl.load(input_ptrs + RANK).to(tl.pointer_type(tl.uint32))
|
||||
input_u32 += input_peer_offset_u32
|
||||
# Slot metadata and the cache destination do not depend on input publish.
|
||||
slot = tl.load(slot_ptr + token)
|
||||
valid = slot >= 0
|
||||
channel = tl.arange(0, CS_P2)
|
||||
channel_mask = channel < CSS
|
||||
global_channel = split * CSS + channel
|
||||
head = tl.minimum(global_channel // HEAD_SIZE, CS // HEAD_SIZE - 1)
|
||||
dim = CACHE_OFFSET + global_channel % HEAD_SIZE
|
||||
safe_slot = tl.maximum(slot, 0).to(tl.int64)
|
||||
dst = (
|
||||
cache_ptr
|
||||
+ (safe_slot // block_size) * stride_cache_block
|
||||
+ head * stride_cache_head
|
||||
+ (safe_slot % block_size) * stride_cache_token
|
||||
+ dim * stride_cache_dim
|
||||
)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
packed = _wait_pairs(input_u32, offsets, mask)
|
||||
|
||||
lo = (packed & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True)
|
||||
hi = (packed >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True)
|
||||
reduced = tl.interleave(
|
||||
tl.sum(lo.to(tl.float32), axis=0).to(tl.bfloat16),
|
||||
tl.sum(hi.to(tl.float32), axis=0).to(tl.bfloat16),
|
||||
)
|
||||
tl.store(
|
||||
input_u32 + offsets,
|
||||
tl.full([WORLD, CS_P2 // 2], _EMPTY_PAIR, tl.uint32),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
tl.store(dst, reduced, mask=valid & channel_mask)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sconv_publish_kernel(
|
||||
peer_ptrs,
|
||||
peer_offset_u32,
|
||||
residual_ptr,
|
||||
weight_ptr,
|
||||
cache_ptr,
|
||||
position_ptr,
|
||||
sequence_ptr,
|
||||
slot_ptr,
|
||||
block_table_ptr,
|
||||
stride_residual_t,
|
||||
stride_cache_block,
|
||||
stride_cache_head,
|
||||
stride_cache_token,
|
||||
stride_cache_dim,
|
||||
stride_block_table_r,
|
||||
max_blocks,
|
||||
block_size,
|
||||
C: tl.constexpr,
|
||||
CS: tl.constexpr,
|
||||
CS_P2: tl.constexpr,
|
||||
SPLITS: tl.constexpr,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
CACHE_OFFSET: tl.constexpr,
|
||||
RANK: tl.constexpr,
|
||||
WORLD: tl.constexpr,
|
||||
WINDOW: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""Compute this rank's shard, then publish it to every rank."""
|
||||
tl.static_assert(WINDOW == 4)
|
||||
token = tl.program_id(0).to(tl.int64)
|
||||
split = tl.program_id(1).to(tl.int64)
|
||||
CSS: tl.constexpr = CS // SPLITS
|
||||
channel = tl.arange(0, CS_P2)
|
||||
channel_mask = channel < CSS
|
||||
global_channel = split * CSS + channel
|
||||
head = tl.minimum(global_channel // HEAD_SIZE, CS // HEAD_SIZE - 1)
|
||||
dim = CACHE_OFFSET + global_channel % HEAD_SIZE
|
||||
slot = tl.load(slot_ptr + token)
|
||||
valid = slot >= 0
|
||||
position = tl.load(position_ptr + token)
|
||||
sequence = tl.load(sequence_ptr + token)
|
||||
safe_slot = tl.maximum(slot, 0).to(tl.int64)
|
||||
own_ptr = (
|
||||
cache_ptr
|
||||
+ (safe_slot // block_size) * stride_cache_block
|
||||
+ head * stride_cache_head
|
||||
+ (safe_slot % block_size) * stride_cache_token
|
||||
+ dim * stride_cache_dim
|
||||
)
|
||||
# These loads were made visible before reduce/insert could signal us and
|
||||
# are independent of its cache store. Hoisting them gives PDL useful work
|
||||
# to overlap while retaining the acquire before every cache read.
|
||||
residual = tl.load(
|
||||
residual_ptr + token * stride_residual_t + RANK * CS + global_channel,
|
||||
mask=channel_mask,
|
||||
other=0.0,
|
||||
)
|
||||
weight0 = tl.load(
|
||||
weight_ptr + global_channel * WINDOW,
|
||||
mask=channel_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
weight1 = tl.load(
|
||||
weight_ptr + global_channel * WINDOW + 1,
|
||||
mask=channel_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
weight2 = tl.load(
|
||||
weight_ptr + global_channel * WINDOW + 2,
|
||||
mask=channel_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
weight3 = tl.load(
|
||||
weight_ptr + global_channel * WINDOW + 3,
|
||||
mask=channel_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
ptrs = peer_ptrs.to(tl.pointer_type(tl.uint64))
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
current = tl.load(own_ptr, mask=valid & channel_mask, other=0.0)
|
||||
|
||||
conv = tl.zeros([CS_P2], tl.float32)
|
||||
for tap_idx in tl.static_range(WINDOW):
|
||||
source_position = position - (WINDOW - 1) + tap_idx
|
||||
take = valid & (source_position >= 0)
|
||||
if tap_idx == WINDOW - 1:
|
||||
value = tl.where(take, current.to(tl.float32), 0.0)
|
||||
else:
|
||||
safe_position = tl.maximum(source_position, 0)
|
||||
logical_block = tl.minimum(safe_position // block_size, max_blocks - 1)
|
||||
physical_block = tl.load(
|
||||
block_table_ptr + sequence * stride_block_table_r + logical_block,
|
||||
mask=take,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
source_ptr = (
|
||||
cache_ptr
|
||||
+ physical_block * stride_cache_block
|
||||
+ head * stride_cache_head
|
||||
+ (safe_position % block_size) * stride_cache_token
|
||||
+ dim * stride_cache_dim
|
||||
)
|
||||
cached = tl.load(source_ptr, mask=take & channel_mask, other=0.0)
|
||||
value = tl.where(take, cached.to(tl.float32), 0.0)
|
||||
if tap_idx == 0:
|
||||
weight = weight0
|
||||
elif tap_idx == 1:
|
||||
weight = weight1
|
||||
elif tap_idx == 2:
|
||||
weight = weight2
|
||||
else:
|
||||
weight = weight3
|
||||
conv += value * weight
|
||||
|
||||
# Preserve both bf16 rounding points of the original sublayer.
|
||||
short_conv_with_skip = (conv + current.to(tl.float32)).to(tl.bfloat16)
|
||||
output = (residual.to(tl.float32) + short_conv_with_skip.to(tl.float32)).to(
|
||||
tl.bfloat16
|
||||
)
|
||||
|
||||
pair = tl.arange(0, CS_P2 // 2)
|
||||
pair_mask = pair < CSS // 2
|
||||
packed = _pack_bf16_pairs(output)
|
||||
row_offset = token * (C // 2) + RANK * (CS // 2) + split * (CSS // 2) + pair
|
||||
|
||||
for destination in tl.static_range(WORLD):
|
||||
base = tl.load(ptrs + destination).to(tl.pointer_type(tl.uint32))
|
||||
tl.store(base + peer_offset_u32 + row_offset, packed, mask=pair_mask)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_norm_kernel(
|
||||
output_peer_ptrs,
|
||||
output_peer_offset_u32,
|
||||
norm_weight_ptr,
|
||||
normed_ptr,
|
||||
residual_out_ptr,
|
||||
eps,
|
||||
stride_output_t,
|
||||
C: tl.constexpr,
|
||||
C_P2: tl.constexpr,
|
||||
RANK: tl.constexpr,
|
||||
HAS_NORM: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""Consume a complete row; one CTA owns all outputs for one token."""
|
||||
token = tl.program_id(0).to(tl.int64)
|
||||
pair = tl.arange(0, C_P2 // 2)
|
||||
pair_mask = pair < C // 2
|
||||
output_ptrs = output_peer_ptrs.to(tl.pointer_type(tl.uint64))
|
||||
output_u32 = tl.load(output_ptrs + RANK).to(tl.pointer_type(tl.uint32))
|
||||
output_u32 += output_peer_offset_u32
|
||||
offsets = token * (C // 2) + pair
|
||||
channel = tl.arange(0, C_P2)
|
||||
channel_mask = channel < C
|
||||
# Gamma is independent of the preceding sconv publication. Keep the
|
||||
# acquire immediately before polling the Lamport output slots.
|
||||
if HAS_NORM:
|
||||
weight = tl.load(norm_weight_ptr + channel, mask=channel_mask, other=0.0)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
packed = _wait_pairs(output_u32, offsets, pair_mask)
|
||||
row = _unpack_bf16_pairs(packed)
|
||||
tl.store(
|
||||
residual_out_ptr + token * stride_output_t + channel, row, mask=channel_mask
|
||||
)
|
||||
if HAS_NORM:
|
||||
row_f32 = tl.where(channel_mask, row.to(tl.float32), 0.0)
|
||||
inv_rms = tl.rsqrt(tl.sum(row_f32 * row_f32, axis=0) / C + eps)
|
||||
tl.store(
|
||||
normed_ptr + token * stride_output_t + channel,
|
||||
(row_f32 * inv_rms * weight.to(tl.float32)).to(tl.bfloat16),
|
||||
mask=channel_mask,
|
||||
)
|
||||
tl.store(
|
||||
output_u32 + offsets,
|
||||
tl.full([C_P2 // 2], _EMPTY_PAIR, tl.uint32),
|
||||
mask=pair_mask,
|
||||
)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _validate_lamport_init_kernel(
|
||||
input_peer_ptrs,
|
||||
output_peer_ptrs,
|
||||
bad_ptr,
|
||||
num_pairs,
|
||||
RANK: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""Validate both complete local allocations through their fabric pointers."""
|
||||
offsets = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offsets < num_pairs
|
||||
ptrs_in = input_peer_ptrs.to(tl.pointer_type(tl.uint64))
|
||||
ptrs_out = output_peer_ptrs.to(tl.pointer_type(tl.uint64))
|
||||
local_in = tl.load(ptrs_in + RANK).to(tl.pointer_type(tl.uint32))
|
||||
local_out = tl.load(ptrs_out + RANK).to(tl.pointer_type(tl.uint32))
|
||||
value_in = tl.load(local_in + offsets, mask=mask, other=_EMPTY_PAIR)
|
||||
value_out = tl.load(local_out + offsets, mask=mask, other=_EMPTY_PAIR)
|
||||
bad = tl.max(
|
||||
tl.where(mask & ((value_in != _EMPTY_PAIR) | (value_out != _EMPTY_PAIR)), 1, 0)
|
||||
)
|
||||
if bad != 0:
|
||||
tl.atomic_max(bad_ptr, 1)
|
||||
|
||||
|
||||
class LamportRSConv:
|
||||
"""Persistent symmetric buffers for one TP group."""
|
||||
|
||||
def _initialize_lamport_buffers(self) -> None:
|
||||
"""Arm every slot and collectively verify the exact sentinel bits.
|
||||
|
||||
Do not replace this with a zero-fill: Lamport readiness distinguishes
|
||||
the bf16 bit pattern for -0.0 (0x8000) from every published payload.
|
||||
The validation is deliberately collective so one rank cannot enter the
|
||||
first polling kernel while another rank still has an unarmed buffer.
|
||||
"""
|
||||
if not self.buf_in.is_contiguous() or not self.buf_out.is_contiguous():
|
||||
raise RuntimeError("Lamport symmetric buffers must be contiguous")
|
||||
if self.buf_in.numel() % 2 or self.buf_out.numel() % 2:
|
||||
raise RuntimeError("Lamport buffers must contain whole uint32 pairs")
|
||||
|
||||
# Fill through int16 so each bf16 lane receives the exact -0.0 bits.
|
||||
# This covers all three generations and all max-token slots, including
|
||||
# slots that a smaller first invocation does not touch.
|
||||
self.buf_in.view(torch.int16).fill_(-0x8000)
|
||||
self.buf_out.view(torch.int16).fill_(-0x8000)
|
||||
torch.accelerator.synchronize(self.device)
|
||||
self.tp.barrier()
|
||||
|
||||
# Scan the complete local allocations. Since every rank performs the
|
||||
# scan and participates in MAX, success proves every symmetric backing
|
||||
# allocation was armed before any rank is allowed to use generation 0.
|
||||
bad = torch.logical_or(
|
||||
self.buf_in.view(torch.int16).ne(-0x8000).any(),
|
||||
self.buf_out.view(torch.int16).ne(-0x8000).any(),
|
||||
).to(dtype=torch.float32)
|
||||
bad = self.tp.all_reduce(bad)
|
||||
if int(bad.item()) != 0:
|
||||
raise RuntimeError("Lamport sentinel initialization failed on a TP rank")
|
||||
self.tp.barrier()
|
||||
|
||||
def _initialize_mnnvl_buffers(self) -> None:
|
||||
"""Initialize and validate FlashInfer fabric-mapped Lamport storage."""
|
||||
self._mnnvl_input_handle.lamport_initialize(self.rank, torch.bfloat16)
|
||||
self._mnnvl_output_handle.lamport_initialize(self.rank, torch.bfloat16)
|
||||
torch.accelerator.synchronize(self.device)
|
||||
self.tp.barrier()
|
||||
|
||||
bad = torch.zeros((), dtype=torch.int32, device=self.device)
|
||||
num_pairs = self.num_buffers * self.max_tokens * self.hidden_size // 2
|
||||
_validate_lamport_init_kernel[(triton.cdiv(num_pairs, 256),)](
|
||||
self.input_peer_ptrs,
|
||||
self.output_peer_ptrs,
|
||||
bad,
|
||||
num_pairs,
|
||||
RANK=self.rank,
|
||||
BLOCK=256,
|
||||
num_warps=4,
|
||||
)
|
||||
bad = self.tp.all_reduce(bad.to(torch.float32))
|
||||
if int(bad.item()) != 0:
|
||||
raise RuntimeError("MNNVL Lamport sentinel initialization failed")
|
||||
self.tp.barrier()
|
||||
|
||||
def __init__(
|
||||
self, hidden_size: int, window_size: int, max_tokens: int = _MAX_TOKENS
|
||||
) -> None:
|
||||
import torch.distributed._symmetric_memory as symm_mem
|
||||
|
||||
tp = get_tp_group()
|
||||
self.tp = tp
|
||||
self.group = tp.device_group
|
||||
self.world_size = tp.world_size
|
||||
self.rank = tp.rank_in_group
|
||||
self.device = torch.device(tp.device)
|
||||
if self.world_size not in (2, 4, 8):
|
||||
raise ValueError(f"TP world size must be 2, 4, or 8, got {self.world_size}")
|
||||
if hidden_size % (2 * self.world_size) != 0:
|
||||
raise ValueError("hidden size must produce an even shard on every rank")
|
||||
if window_size != 4:
|
||||
raise ValueError(f"short-conv window size must be 4, got {window_size}")
|
||||
if max_tokens < 1 or max_tokens > _MAX_TOKENS:
|
||||
raise ValueError(f"max_tokens must be in [1, {_MAX_TOKENS}]")
|
||||
|
||||
is_cross_node = not all(in_the_same_node_as(tp.cpu_group))
|
||||
self.hidden_size = hidden_size
|
||||
self.window_size = window_size
|
||||
self.max_tokens = max_tokens
|
||||
self.shard_size = hidden_size // self.world_size
|
||||
# Three generations follow FlashInfer's Lamport layout. A generation
|
||||
# is reused only after two intervening collective calls.
|
||||
self.num_buffers = 3
|
||||
self.input_generation_bytes = max_tokens * hidden_size * 2
|
||||
self.output_generation_bytes = max_tokens * hidden_size * 2
|
||||
self.use_pdl = torch.cuda.get_device_capability(self.device)[0] >= 9
|
||||
if is_cross_node:
|
||||
try:
|
||||
from flashinfer.comm.mnnvl import (
|
||||
McastGPUBuffer,
|
||||
TorchDistBackend,
|
||||
is_mnnvl_fabric_supported,
|
||||
)
|
||||
except ImportError as error:
|
||||
raise RuntimeError(
|
||||
"cross-node TP requires FlashInfer MNNVL support"
|
||||
) from error
|
||||
|
||||
local_supported = int(
|
||||
is_mnnvl_fabric_supported(torch.accelerator.current_device_index())
|
||||
)
|
||||
unsupported = torch.tensor(
|
||||
1 - local_supported, dtype=torch.float32, device=self.device
|
||||
)
|
||||
unsupported = tp.all_reduce(unsupported)
|
||||
if int(unsupported.item()) != 0:
|
||||
raise RuntimeError("cross-node TP is supported only on MNNVL fabric")
|
||||
|
||||
comm_backend = TorchDistBackend(self.group)
|
||||
allocation_bytes = self.num_buffers * max_tokens * hidden_size * 2
|
||||
self._mnnvl_input_handle = McastGPUBuffer(
|
||||
allocation_bytes,
|
||||
self.world_size,
|
||||
self.rank,
|
||||
self.device,
|
||||
comm_backend,
|
||||
)
|
||||
self._mnnvl_output_handle = McastGPUBuffer(
|
||||
allocation_bytes,
|
||||
self.world_size,
|
||||
self.rank,
|
||||
self.device,
|
||||
comm_backend,
|
||||
)
|
||||
self.input_peer_ptrs = self._mnnvl_input_handle.get_buffer_ptrs_dev()
|
||||
self.output_peer_ptrs = self._mnnvl_output_handle.get_buffer_ptrs_dev()
|
||||
self._initialize_mnnvl_buffers()
|
||||
logger.info("using FlashInfer fabric-mapped MNNVL Lamport buffers")
|
||||
else:
|
||||
self.buf_in = symm_mem.empty(
|
||||
self.num_buffers,
|
||||
max_tokens,
|
||||
self.world_size,
|
||||
self.shard_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=self.device,
|
||||
)
|
||||
self.buf_out = symm_mem.empty(
|
||||
self.num_buffers,
|
||||
max_tokens,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=self.device,
|
||||
)
|
||||
group_name = self.group.group_name
|
||||
input_handle = symm_mem.rendezvous(self.buf_in, group_name)
|
||||
output_handle = symm_mem.rendezvous(self.buf_out, group_name)
|
||||
self.input_peer_ptrs = input_handle.buffer_ptrs_dev
|
||||
self.output_peer_ptrs = output_handle.buffer_ptrs_dev
|
||||
self._input_handle = input_handle
|
||||
self._output_handle = output_handle
|
||||
self._initialize_lamport_buffers()
|
||||
self.generation = 0
|
||||
|
||||
def usable(self, num_tokens: int) -> bool:
|
||||
return 0 < num_tokens <= self.max_tokens
|
||||
|
||||
def rs_sconv_ag_add_norm(
|
||||
self,
|
||||
input_tensor: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
norm_weight: torch.Tensor | None,
|
||||
eps: float,
|
||||
cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
seq_idx: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
off_s: int,
|
||||
ws: int,
|
||||
block_size: int,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""Return ``(normed | None, new_residual)``, both shaped ``[T, 6144]``."""
|
||||
tokens, hidden_size = residual.shape
|
||||
if not self.usable(tokens):
|
||||
raise ValueError(f"num_tokens must be in [1, {self.max_tokens}]")
|
||||
if hidden_size != self.hidden_size or residual.dtype != torch.bfloat16:
|
||||
raise ValueError("residual must be bf16 [T, 6144]")
|
||||
if (
|
||||
input_tensor.shape != residual.shape
|
||||
or input_tensor.dtype != torch.bfloat16
|
||||
or input_tensor.stride(1) != 1
|
||||
):
|
||||
raise ValueError("input_tensor must be channel-contiguous bf16 [T, 6144]")
|
||||
shard_size = hidden_size // self.world_size
|
||||
if conv_weight.shape != (shard_size, self.window_size):
|
||||
raise ValueError(
|
||||
f"conv_weight must have shape [{shard_size}, {self.window_size}]"
|
||||
)
|
||||
if (
|
||||
conv_weight.dtype != torch.bfloat16
|
||||
or conv_weight.stride(0) != self.window_size
|
||||
):
|
||||
raise ValueError("conv_weight must be contiguous bf16")
|
||||
if norm_weight is not None and (
|
||||
norm_weight.shape != (hidden_size,) or norm_weight.dtype != torch.bfloat16
|
||||
):
|
||||
raise ValueError("norm_weight must be bf16 [6144] or None")
|
||||
if cache.dtype != torch.bfloat16 or cache.ndim != 4:
|
||||
raise ValueError("cache must be a 4-D bf16 tensor")
|
||||
if shard_size % ws != 0 or cache.shape[1] != shard_size // ws:
|
||||
raise ValueError("cache head layout is inconsistent with ws")
|
||||
if off_s < 0 or off_s + ws > cache.shape[3]:
|
||||
raise ValueError("cache channel offset is out of bounds")
|
||||
|
||||
index = self.generation
|
||||
input_offset = index * self.input_generation_bytes // 4
|
||||
output_offset = index * self.output_generation_bytes // 4
|
||||
normed = torch.empty_like(residual) if norm_weight is not None else None
|
||||
residual_out = torch.empty_like(residual)
|
||||
phase_splits = 8 if tokens == 1 and shard_size % 8 == 0 else 1
|
||||
phase_tile_p2 = triton.next_power_of_2(shard_size // phase_splits)
|
||||
phase_grid = (tokens, phase_splits)
|
||||
# Wide CTAs win before the grid saturates; smaller CTAs reduce register
|
||||
# pressure once high-throughput batches provide enough parallelism.
|
||||
if 128 <= tokens <= 2048:
|
||||
phase_warps = 16
|
||||
elif tokens > 2048:
|
||||
phase_warps = 8
|
||||
else:
|
||||
phase_warps = 4
|
||||
gather_warps = 4 if tokens >= 256 else 8
|
||||
_publish_input_kernel[phase_grid](
|
||||
input_tensor,
|
||||
self.input_peer_ptrs,
|
||||
input_offset,
|
||||
input_tensor.stride(0),
|
||||
C=hidden_size,
|
||||
CS=shard_size,
|
||||
CS_P2=phase_tile_p2,
|
||||
SPLITS=phase_splits,
|
||||
RANK=self.rank,
|
||||
WORLD=self.world_size,
|
||||
USE_PDL=self.use_pdl,
|
||||
launch_pdl=self.use_pdl,
|
||||
num_warps=phase_warps,
|
||||
)
|
||||
_reduce_insert_kernel[phase_grid](
|
||||
self.input_peer_ptrs,
|
||||
input_offset,
|
||||
cache,
|
||||
slot_mapping,
|
||||
cache.stride(0),
|
||||
cache.stride(1),
|
||||
cache.stride(2),
|
||||
cache.stride(3),
|
||||
block_size,
|
||||
C=hidden_size,
|
||||
CS=shard_size,
|
||||
CS_P2=phase_tile_p2,
|
||||
SPLITS=phase_splits,
|
||||
HEAD_SIZE=ws,
|
||||
CACHE_OFFSET=off_s,
|
||||
RANK=self.rank,
|
||||
WORLD=self.world_size,
|
||||
USE_PDL=self.use_pdl,
|
||||
launch_pdl=self.use_pdl,
|
||||
num_warps=phase_warps,
|
||||
)
|
||||
_sconv_publish_kernel[phase_grid](
|
||||
self.output_peer_ptrs,
|
||||
output_offset,
|
||||
residual,
|
||||
conv_weight,
|
||||
cache,
|
||||
positions,
|
||||
seq_idx,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
residual.stride(0),
|
||||
cache.stride(0),
|
||||
cache.stride(1),
|
||||
cache.stride(2),
|
||||
cache.stride(3),
|
||||
block_table.stride(0),
|
||||
block_table.shape[1],
|
||||
block_size,
|
||||
C=hidden_size,
|
||||
CS=shard_size,
|
||||
CS_P2=phase_tile_p2,
|
||||
SPLITS=phase_splits,
|
||||
HEAD_SIZE=ws,
|
||||
CACHE_OFFSET=off_s,
|
||||
RANK=self.rank,
|
||||
WORLD=self.world_size,
|
||||
WINDOW=self.window_size,
|
||||
USE_PDL=self.use_pdl,
|
||||
launch_pdl=self.use_pdl,
|
||||
num_warps=phase_warps,
|
||||
)
|
||||
_gather_norm_kernel[(tokens,)](
|
||||
self.output_peer_ptrs,
|
||||
output_offset,
|
||||
norm_weight if norm_weight is not None else residual,
|
||||
normed if normed is not None else residual_out,
|
||||
residual_out,
|
||||
eps,
|
||||
residual.stride(0),
|
||||
C=hidden_size,
|
||||
C_P2=triton.next_power_of_2(hidden_size),
|
||||
RANK=self.rank,
|
||||
HAS_NORM=norm_weight is not None,
|
||||
USE_PDL=self.use_pdl,
|
||||
launch_pdl=self.use_pdl,
|
||||
num_warps=gather_warps,
|
||||
)
|
||||
self.generation = (index + 1) % self.num_buffers
|
||||
return normed, residual_out
|
||||
|
||||
|
||||
_STATE: LamportRSConv | None = None
|
||||
_STATE_FAILED = False
|
||||
|
||||
|
||||
def initialize_lamport_rs_conv(
|
||||
hidden_size: int, window_size: int, max_num_batched_tokens: int
|
||||
) -> None:
|
||||
"""Collectively initialize the TP-group state during model construction."""
|
||||
global _STATE, _STATE_FAILED
|
||||
if _STATE is not None:
|
||||
if _STATE.hidden_size != hidden_size or _STATE.window_size != window_size:
|
||||
raise RuntimeError("all Lamport users must share hidden and window sizes")
|
||||
return
|
||||
if _STATE_FAILED or os.environ.get("LAMPORT_RS_SCONV", "1") == "0":
|
||||
return
|
||||
try:
|
||||
max_tokens = min(_MAX_TOKENS, max_num_batched_tokens)
|
||||
_STATE = LamportRSConv(hidden_size, window_size, max_tokens=max_tokens)
|
||||
except Exception:
|
||||
_STATE_FAILED = True
|
||||
logger.exception("fused collective unavailable; use the NCCL fallback")
|
||||
|
||||
|
||||
def get_lamport_rs_conv(hidden_size: int, window_size: int) -> LamportRSConv | None:
|
||||
"""Return the state initialized with the model, or ``None`` for fallback."""
|
||||
if _STATE is not None and (
|
||||
_STATE.hidden_size != hidden_size or _STATE.window_size != window_size
|
||||
):
|
||||
raise RuntimeError("all Lamport users must share hidden and window sizes")
|
||||
return _STATE
|
||||
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused CUDA kernels for the Inkling vision/audio towers.
|
||||
|
||||
Both kernels keep the reference paths' fp32 accumulation and per-op bf16
|
||||
rounding points (native ``rms_norm`` / ``F.gelu``); outputs are frequently
|
||||
bit-identical and otherwise differ by 1-2 bf16 ulps from reduction-order
|
||||
(real-checkpoint-weight cosine vs reference > 0.9999998).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, tldevice, triton
|
||||
|
||||
from .norm import _get_num_warps_from_block_size
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dmel_embed_sum_norm_kernel(
|
||||
idx_ptr, # [T, NB] int32 dMel bin indices (values in [0, VOCAB))
|
||||
w_ptr, # [NB * VOCAB, D] bf16 embedding table
|
||||
norm_w_ptr, # [D] (unused if HAS_NORM=False)
|
||||
out_ptr, # [T, D] bf16
|
||||
eps,
|
||||
D,
|
||||
stride_idx_t,
|
||||
NB: tl.constexpr,
|
||||
VOCAB: tl.constexpr,
|
||||
D_P2: tl.constexpr,
|
||||
HAS_NORM: tl.constexpr,
|
||||
):
|
||||
t = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.arange(0, D_P2)
|
||||
mask = offs < D
|
||||
# One embedding row per mel bin (bin b uses table rows [b*VOCAB, (b+1)*VOCAB)),
|
||||
# summed in fp32 (matches torch's fp32-accumulated bf16 .sum()).
|
||||
acc = tl.zeros([D_P2], dtype=tl.float32)
|
||||
for b in tl.static_range(NB):
|
||||
v = tl.load(idx_ptr + t * stride_idx_t + b)
|
||||
row = (b * VOCAB + v).to(tl.int64)
|
||||
acc += tl.load(w_ptr + row * D + offs, mask=mask, other=0.0).to(tl.float32)
|
||||
h = acc.to(tl.bfloat16)
|
||||
if HAS_NORM:
|
||||
# Match ir.ops.rms_norm: fp32 variance/normalize, then a single-rounded
|
||||
# bf16 multiply with the bf16 weight.
|
||||
x32 = h.to(tl.float32)
|
||||
var = tl.sum(tl.where(mask, x32 * x32, 0.0), axis=0) / D
|
||||
xn = (x32 * tl.math.rsqrt(var + eps)).to(tl.bfloat16)
|
||||
w = tl.load(norm_w_ptr + offs, mask=mask, other=0.0)
|
||||
h = xn * w
|
||||
tl.store(out_ptr + t * D + offs, h, mask=mask)
|
||||
|
||||
|
||||
def dmel_embed_sum_norm(
|
||||
dmel_idx: torch.Tensor, # [T, NB] int32
|
||||
weight: torch.Tensor, # [NB * VOCAB, D] bf16
|
||||
norm_weight: torch.Tensor | None,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""``rmsnorm(sum_b weight[b * VOCAB + idx[:, b]])`` in one launch (no
|
||||
[T, NB, D] intermediate)."""
|
||||
T, nb = dmel_idx.shape
|
||||
D = weight.shape[1]
|
||||
assert weight.shape[0] % nb == 0
|
||||
vocab = weight.shape[0] // nb
|
||||
out = torch.empty((T, D), dtype=weight.dtype, device=weight.device)
|
||||
if T == 0:
|
||||
return out
|
||||
d_p2 = triton.next_power_of_2(D)
|
||||
_dmel_embed_sum_norm_kernel[(T,)](
|
||||
dmel_idx,
|
||||
weight,
|
||||
norm_weight if norm_weight is not None else weight,
|
||||
out,
|
||||
eps,
|
||||
D,
|
||||
dmel_idx.stride(0),
|
||||
NB=nb,
|
||||
VOCAB=vocab,
|
||||
D_P2=d_p2,
|
||||
HAS_NORM=norm_weight is not None,
|
||||
# Swept on GB200: 8 warps beats the block-size heuristic's 16 by ~4%
|
||||
# at large T (the 80-row gather chain is latency- not lane-bound).
|
||||
num_warps=8,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rmsnorm_gelu_kernel(
|
||||
x_ptr, # [R, D] bf16
|
||||
w_ptr, # [D]
|
||||
out_ptr, # [R, D] bf16 (or the folded layout when FOLD)
|
||||
eps,
|
||||
R,
|
||||
D,
|
||||
D_P2: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr, # rows per block (>1 for small D)
|
||||
HAS_GELU: tl.constexpr,
|
||||
FOLD: tl.constexpr,
|
||||
# fold geometry: input rows index [N, T, H, W]; the store scatters each
|
||||
# row to (out_row, slot) of fold_timespace_to_depth's output layout.
|
||||
FT: tl.constexpr,
|
||||
FH: tl.constexpr,
|
||||
FW: tl.constexpr,
|
||||
TF: tl.constexpr,
|
||||
HF: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0).to(tl.int64)
|
||||
rows = pid * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
rmask = rows < R
|
||||
offs = tl.arange(0, D_P2)
|
||||
mask = rmask[:, None] & (offs < D)[None, :]
|
||||
x32 = tl.load(x_ptr + rows[:, None] * D + offs[None, :], mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
var = tl.sum(x32 * x32, axis=1) / D
|
||||
xn = (x32 * tl.math.rsqrt(var + eps)[:, None]).to(tl.bfloat16)
|
||||
w = tl.load(w_ptr + offs, mask=offs < D, other=0.0)
|
||||
h = xn * w[None, :] # bf16 multiply, matching ir.ops.rms_norm
|
||||
if HAS_GELU:
|
||||
# Exact (erf) GELU on the bf16-rounded norm output, fp32 math,
|
||||
# matching F.gelu's opmath on a bf16 tensor.
|
||||
g32 = h.to(tl.float32)
|
||||
h = (0.5 * g32 * (1.0 + tldevice.erf(g32 * 0.7071067811865476))).to(tl.bfloat16)
|
||||
if FOLD:
|
||||
# Store directly into the next layer's folded layout (a pure
|
||||
# permutation — replaces the separate fold copy pass).
|
||||
t = (rows // (FH * FW)) % FT
|
||||
hh = (rows // FW) % FH
|
||||
ww = rows % FW
|
||||
n = rows // (FT * FH * FW)
|
||||
slot = ((t % TF) * HF + hh % HF) * HF + ww % HF
|
||||
out_row = ((n * (FT // TF) + t // TF) * (FH // HF) + hh // HF) * (
|
||||
FW // HF
|
||||
) + ww // HF
|
||||
base = (out_row * (TF * HF * HF) + slot) * D
|
||||
tl.store(out_ptr + base[:, None] + offs[None, :], h, mask=mask)
|
||||
else:
|
||||
tl.store(out_ptr + rows[:, None] * D + offs[None, :], h, mask=mask)
|
||||
|
||||
|
||||
def rmsnorm_gelu(
|
||||
x: torch.Tensor, # [..., D] bf16 contiguous
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
gelu: bool = True,
|
||||
fold: tuple[int, int] | None = None, # (t_fold, hw_fold) of the NEXT fold
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``gelu(rmsnorm(x))`` (or plain rmsnorm); multiple rows per block
|
||||
when D is small. With ``fold``, x must be [N, T, H, W, D] and the output
|
||||
comes back as ``fold_timespace_to_depth(result, *fold)``."""
|
||||
D = x.shape[-1]
|
||||
flat = x.reshape(-1, D)
|
||||
assert flat.stride(1) == 1 and flat.stride(0) == D
|
||||
R = flat.shape[0]
|
||||
if fold is None:
|
||||
out = torch.empty_like(flat)
|
||||
ft = fh = fw = tf = hf = 1
|
||||
out_shape = x.shape
|
||||
else:
|
||||
tf, hf = fold
|
||||
N, ft, fh, fw, _ = x.shape
|
||||
out_shape = (N, ft // tf, fh // hf, fw // hf, tf * hf * hf * D)
|
||||
out = torch.empty(out_shape, dtype=x.dtype, device=x.device)
|
||||
if R == 0:
|
||||
return out.reshape(out_shape)
|
||||
d_p2 = triton.next_power_of_2(D)
|
||||
block_m = max(1, 4096 // d_p2)
|
||||
_rmsnorm_gelu_kernel[(triton.cdiv(R, block_m),)](
|
||||
flat,
|
||||
weight,
|
||||
out,
|
||||
eps,
|
||||
R,
|
||||
D,
|
||||
D_P2=d_p2,
|
||||
BLOCK_M=block_m,
|
||||
HAS_GELU=gelu,
|
||||
FOLD=fold is not None,
|
||||
FT=ft,
|
||||
FH=fh,
|
||||
FW=fw,
|
||||
TF=tf,
|
||||
HF=hf,
|
||||
num_warps=_get_num_warps_from_block_size(d_p2 * block_m),
|
||||
)
|
||||
return out.reshape(out_shape)
|
||||
@@ -0,0 +1,414 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_MAX_FUSED_SIZE = 65536
|
||||
|
||||
|
||||
def _get_num_warps_from_block_size(block_size: int) -> int:
|
||||
if block_size >= 32768:
|
||||
return 32
|
||||
if block_size >= 8192:
|
||||
return 16
|
||||
if block_size >= 2048:
|
||||
return 8
|
||||
return 4
|
||||
|
||||
|
||||
def _largest_power_of_2(n: int) -> int:
|
||||
assert n > 0, f"{n=}"
|
||||
return 1 << (n.bit_length() - 1)
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def _get_grid_size_for_mem_bw_kernel(device: torch.device, factor: int = 8) -> int:
|
||||
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
|
||||
return _largest_power_of_2(num_sms) * factor
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rmsnorm_fwd_kernel(
|
||||
x_ptr,
|
||||
weight_ptr,
|
||||
y_ptr,
|
||||
rstd_ptr,
|
||||
eps,
|
||||
x_stride_0,
|
||||
y_stride_0,
|
||||
n_cols,
|
||||
block_size_n: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0).to(tl.int64)
|
||||
offs_n = tl.arange(0, block_size_n)
|
||||
mask_n = offs_n < n_cols
|
||||
|
||||
weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
|
||||
x = tl.load(x_ptr + pid_m * x_stride_0 + offs_n, mask=mask_n, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
row_var = tl.sum(x * x, axis=0) / n_cols
|
||||
rstd = tl.math.rsqrt(row_var + eps)
|
||||
tl.store(rstd_ptr + pid_m, rstd)
|
||||
|
||||
y = x * rstd * weight
|
||||
tl.store(y_ptr + pid_m * y_stride_0 + offs_n, y, mask=mask_n)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["n_rows"])
|
||||
def _rmsnorm_fwd_kernel_block_m(
|
||||
x_ptr,
|
||||
weight_ptr,
|
||||
y_ptr,
|
||||
rstd_ptr,
|
||||
eps,
|
||||
x_stride_0,
|
||||
y_stride_0,
|
||||
n_rows,
|
||||
n_cols,
|
||||
block_size_m: tl.constexpr,
|
||||
block_size_n: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0).to(tl.int64)
|
||||
offs_n = tl.arange(0, block_size_n)
|
||||
mask_n = offs_n < n_cols
|
||||
|
||||
num_blocks_m = tl.cdiv(n_rows, block_size_m)
|
||||
blocks_per_pid = tl.cdiv(num_blocks_m, tl.num_programs(0))
|
||||
block_id_start = pid_m * blocks_per_pid
|
||||
block_id_end = min(block_id_start + blocks_per_pid, num_blocks_m)
|
||||
|
||||
weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
|
||||
|
||||
for block_id in range(block_id_start, block_id_end):
|
||||
offs_m = block_id * block_size_m + tl.arange(0, block_size_m)
|
||||
mask_m = offs_m < n_rows
|
||||
mask_mn = mask_m[:, None] & mask_n[None, :]
|
||||
x = tl.load(
|
||||
x_ptr + offs_m[:, None] * x_stride_0 + offs_n[None, :],
|
||||
mask=mask_mn,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
row_var = tl.sum(x * x, axis=1) / n_cols
|
||||
rstd = tl.math.rsqrt(row_var + eps)
|
||||
tl.store(rstd_ptr + offs_m, rstd, mask=mask_m)
|
||||
|
||||
y = x * rstd[:, None] * weight
|
||||
tl.store(
|
||||
y_ptr + offs_m[:, None] * y_stride_0 + offs_n[None, :],
|
||||
y,
|
||||
mask=mask_mn,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _add_rmsnorm_fwd_kernel(
|
||||
res_ptr, # [T, N] residual (read)
|
||||
delta_ptr, # [T, N] delta to add (read)
|
||||
weight_ptr,
|
||||
y_ptr, # [T, N] normed output
|
||||
res_out_ptr, # [T, N] updated residual output
|
||||
eps,
|
||||
res_stride_0,
|
||||
delta_stride_0,
|
||||
y_stride_0,
|
||||
res_out_stride_0,
|
||||
n_cols,
|
||||
block_size_n: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0).to(tl.int64)
|
||||
offs_n = tl.arange(0, block_size_n)
|
||||
mask_n = offs_n < n_cols
|
||||
|
||||
weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
|
||||
r = tl.load(res_ptr + pid_m * res_stride_0 + offs_n, mask=mask_n, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
d = tl.load(delta_ptr + pid_m * delta_stride_0 + offs_n, mask=mask_n, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
# Round the sum to the residual dtype first (matches the eager
|
||||
# `residual + delta` then rmsnorm-on-bf16 sequence bit-for-bit).
|
||||
s = (r + d).to(res_out_ptr.dtype.element_ty)
|
||||
tl.store(res_out_ptr + pid_m * res_out_stride_0 + offs_n, s, mask=mask_n)
|
||||
x = s.to(tl.float32)
|
||||
row_var = tl.sum(x * x, axis=0) / n_cols
|
||||
rstd = tl.math.rsqrt(row_var + eps)
|
||||
y = x * rstd * weight
|
||||
tl.store(y_ptr + pid_m * y_stride_0 + offs_n, y, mask=mask_n)
|
||||
|
||||
|
||||
def add_rmsnorm(
|
||||
residual: torch.Tensor,
|
||||
delta: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused ``res = residual + delta; y = rmsnorm(res)``.
|
||||
|
||||
Returns ``(y, res)``; both are fresh tensors (cudagraph-friendly, no
|
||||
in-place update of the inputs).
|
||||
"""
|
||||
assert residual.ndim == 2 and delta.ndim == 2, (residual.shape, delta.shape)
|
||||
n_rows, n_cols = residual.shape
|
||||
assert weight.shape[0] == n_cols
|
||||
y = torch.empty_like(residual)
|
||||
res_out = torch.empty_like(residual)
|
||||
if n_rows == 0:
|
||||
return y, res_out
|
||||
|
||||
block_size_n = triton.next_power_of_2(n_cols)
|
||||
max_block_size_n = _MAX_FUSED_SIZE // residual.element_size()
|
||||
if max_block_size_n < block_size_n:
|
||||
raise RuntimeError(f"Large {n_cols=} is not supported")
|
||||
num_warps = _get_num_warps_from_block_size(block_size_n)
|
||||
_add_rmsnorm_fwd_kernel[(n_rows,)](
|
||||
residual,
|
||||
delta,
|
||||
weight,
|
||||
y,
|
||||
res_out,
|
||||
eps,
|
||||
residual.stride(0),
|
||||
delta.stride(0),
|
||||
y.stride(0),
|
||||
res_out.stride(0),
|
||||
n_cols,
|
||||
block_size_n,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
return y, res_out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _embed_rmsnorm_kernel(
|
||||
ids_ptr, # [T] token ids
|
||||
table_ptr, # [V, N] embedding table
|
||||
weight_ptr, # [N] (HAS_NORM only)
|
||||
chain_weight_ptr, # [N] (HAS_CHAIN only)
|
||||
out_ptr, # [T, N] rmsnorm(table[ids], weight)
|
||||
chain_out_ptr, # [T, N] rmsnorm(out, chain_weight) (HAS_CHAIN only)
|
||||
eps,
|
||||
table_stride_0,
|
||||
n_cols,
|
||||
block_size_n: tl.constexpr,
|
||||
HAS_NORM: tl.constexpr,
|
||||
HAS_CHAIN: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0).to(tl.int64)
|
||||
offs_n = tl.arange(0, block_size_n)
|
||||
mask_n = offs_n < n_cols
|
||||
row = tl.load(ids_ptr + pid_m).to(tl.int64)
|
||||
x = tl.load(table_ptr + row * table_stride_0 + offs_n, mask=mask_n, other=0.0)
|
||||
if HAS_NORM:
|
||||
xf = x.to(tl.float32)
|
||||
rstd = tl.math.rsqrt(tl.sum(xf * xf, axis=0) / n_cols + eps)
|
||||
w = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
|
||||
# Round to the output dtype so the chained norm is bit-exact vs the
|
||||
# unfused pair (which stores bf16 in between).
|
||||
x = (xf * rstd * w).to(out_ptr.dtype.element_ty)
|
||||
tl.store(out_ptr + pid_m * n_cols + offs_n, x, mask=mask_n)
|
||||
if HAS_CHAIN:
|
||||
xf = x.to(tl.float32)
|
||||
rstd = tl.math.rsqrt(tl.sum(xf * xf, axis=0) / n_cols + eps)
|
||||
w = tl.load(chain_weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
|
||||
tl.store(
|
||||
chain_out_ptr + pid_m * n_cols + offs_n,
|
||||
(xf * rstd * w).to(chain_out_ptr.dtype.element_ty),
|
||||
mask=mask_n,
|
||||
)
|
||||
|
||||
|
||||
def embed_rmsnorm(
|
||||
input_ids: torch.Tensor,
|
||||
embed_table: torch.Tensor,
|
||||
weight: torch.Tensor | None,
|
||||
eps: float,
|
||||
chain_weight: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused ``rmsnorm(embed_table[input_ids], weight)`` row gather + norm.
|
||||
|
||||
Requires the full vocab on-rank (replicated or tp_size == 1).
|
||||
``weight=None`` skips the norm (``use_embed_norm=False``), leaving a pure
|
||||
embedding-table gather.
|
||||
``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
|
||||
first decoder layer's pre-attention norm) as a second output, still one
|
||||
launch. Bit-exact vs the unfused module sequence."""
|
||||
ids = input_ids.view(-1)
|
||||
(T,) = ids.shape
|
||||
n = embed_table.shape[1]
|
||||
out = torch.empty(
|
||||
(*input_ids.shape, n), dtype=embed_table.dtype, device=embed_table.device
|
||||
)
|
||||
chain_out = torch.empty_like(out) if chain_weight is not None else None
|
||||
if T > 0:
|
||||
block_size_n = triton.next_power_of_2(n)
|
||||
_embed_rmsnorm_kernel[(T,)](
|
||||
ids,
|
||||
embed_table,
|
||||
weight if weight is not None else embed_table,
|
||||
chain_weight if chain_weight is not None else embed_table,
|
||||
out,
|
||||
chain_out if chain_out is not None else out,
|
||||
eps,
|
||||
embed_table.stride(0),
|
||||
n,
|
||||
block_size_n,
|
||||
HAS_NORM=weight is not None,
|
||||
HAS_CHAIN=chain_weight is not None,
|
||||
num_warps=_get_num_warps_from_block_size(block_size_n),
|
||||
)
|
||||
if chain_out is not None:
|
||||
return out, chain_out
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _embed_dual_rmsnorm_cat_kernel(
|
||||
hidden_ptr, # [T, N]
|
||||
emb_ptr, # [T, N] embeddings, or the [V, N] embedding table when GATHER
|
||||
ids_ptr, # [T] token ids (GATHER only)
|
||||
w_hidden_ptr, # [N]
|
||||
w_pre_ptr, # [N] chained pre-norm on the embed side (HAS_PRE_NORM only)
|
||||
w_embed_ptr, # [N]
|
||||
out_ptr, # [T, 2N]: [rmsnorm(hidden) | rmsnorm(rmsnorm?(emb))]
|
||||
eps,
|
||||
hidden_stride_0,
|
||||
emb_stride_0,
|
||||
n_cols,
|
||||
block_size_n: tl.constexpr,
|
||||
GATHER: tl.constexpr,
|
||||
HAS_PRE_NORM: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0).to(tl.int64)
|
||||
which = tl.program_id(1) # 0 -> hidden into cols [0, N); 1 -> emb into [N, 2N)
|
||||
offs_n = tl.arange(0, block_size_n)
|
||||
mask_n = offs_n < n_cols
|
||||
if which == 0:
|
||||
x = tl.load(
|
||||
hidden_ptr + pid_m * hidden_stride_0 + offs_n, mask=mask_n, other=0.0
|
||||
).to(tl.float32)
|
||||
w = tl.load(w_hidden_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
|
||||
else:
|
||||
row = tl.load(ids_ptr + pid_m).to(tl.int64) if GATHER else pid_m
|
||||
x = tl.load(emb_ptr + row * emb_stride_0 + offs_n, mask=mask_n, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
if HAS_PRE_NORM:
|
||||
w_pre = tl.load(w_pre_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
|
||||
rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / n_cols + eps)
|
||||
# Round-trip through the output dtype so the chained norm is
|
||||
# bit-exact vs the unfused pair (which stores bf16 in between).
|
||||
x = (x * rstd * w_pre).to(out_ptr.dtype.element_ty).to(tl.float32)
|
||||
w = tl.load(w_embed_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
|
||||
rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / n_cols + eps)
|
||||
tl.store(
|
||||
out_ptr + pid_m * (2 * n_cols) + which * n_cols + offs_n,
|
||||
(x * rstd * w).to(out_ptr.dtype.element_ty),
|
||||
mask=mask_n,
|
||||
)
|
||||
|
||||
|
||||
def embed_dual_rmsnorm_cat(
|
||||
hidden: torch.Tensor,
|
||||
hidden_weight: torch.Tensor,
|
||||
embed_weight: torch.Tensor,
|
||||
eps: float,
|
||||
*,
|
||||
embeds: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
embed_table: torch.Tensor | None = None,
|
||||
pre_norm_weight: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""The MTP depth-layer input in one launch:
|
||||
``cat([rmsnorm(hidden, w_h), rmsnorm(pre?(emb), w_e)], -1)``.
|
||||
|
||||
The embed side is either a fused row gather ``embed_table[input_ids]``
|
||||
(draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged
|
||||
multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the
|
||||
backbone embed_norm in front of the depth embed_norm (bit-exact vs the
|
||||
unfused sequence). The concat copies collapse into direct writes."""
|
||||
T, n = hidden.shape
|
||||
if embeds is not None:
|
||||
assert embeds.shape == hidden.shape
|
||||
src, ids, src_stride = embeds, embeds, embeds.stride(0)
|
||||
gather = False
|
||||
else:
|
||||
assert input_ids is not None and embed_table is not None
|
||||
assert input_ids.shape == (T,) and embed_table.shape[1] == n
|
||||
src, ids, src_stride = embed_table, input_ids, embed_table.stride(0)
|
||||
gather = True
|
||||
out = torch.empty((T, 2 * n), dtype=hidden.dtype, device=hidden.device)
|
||||
if T == 0:
|
||||
return out
|
||||
block_size_n = triton.next_power_of_2(n)
|
||||
_embed_dual_rmsnorm_cat_kernel[(T, 2)](
|
||||
hidden,
|
||||
src,
|
||||
ids,
|
||||
hidden_weight,
|
||||
pre_norm_weight if pre_norm_weight is not None else embed_weight,
|
||||
embed_weight,
|
||||
out,
|
||||
eps,
|
||||
hidden.stride(0),
|
||||
src_stride,
|
||||
n,
|
||||
block_size_n,
|
||||
GATHER=gather,
|
||||
HAS_PRE_NORM=pre_norm_weight is not None,
|
||||
num_warps=_get_num_warps_from_block_size(block_size_n),
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
assert x.ndim == 2, f"{x.shape=}"
|
||||
assert weight.ndim == 1, f"{weight.shape=}"
|
||||
n_rows, n_cols = x.shape
|
||||
assert weight.shape[0] == n_cols, f"{weight.shape=} {x.shape=}"
|
||||
y = torch.empty_like(x)
|
||||
rstd = torch.empty((n_rows,), dtype=torch.float32, device=x.device)
|
||||
|
||||
block_size_n = triton.next_power_of_2(n_cols)
|
||||
max_block_size_n = _MAX_FUSED_SIZE // x.element_size()
|
||||
if max_block_size_n < block_size_n:
|
||||
raise RuntimeError(f"Large {n_cols=} is not supported")
|
||||
block_size_m = max(1, 4096 // block_size_n)
|
||||
num_warps = _get_num_warps_from_block_size(block_size_n)
|
||||
|
||||
if block_size_m == 1:
|
||||
_rmsnorm_fwd_kernel[(n_rows,)](
|
||||
x,
|
||||
weight,
|
||||
y,
|
||||
rstd,
|
||||
eps,
|
||||
x.stride(0),
|
||||
y.stride(0),
|
||||
n_cols,
|
||||
block_size_n,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
else:
|
||||
grid_size = _get_grid_size_for_mem_bw_kernel(x.device)
|
||||
_rmsnorm_fwd_kernel_block_m[(grid_size,)](
|
||||
x,
|
||||
weight,
|
||||
y,
|
||||
rstd,
|
||||
eps,
|
||||
x.stride(0),
|
||||
y.stride(0),
|
||||
n_rows,
|
||||
n_cols,
|
||||
block_size_m,
|
||||
block_size_n,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
return y
|
||||
@@ -0,0 +1,918 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import aux_stream
|
||||
|
||||
LOW_BLOCK_M = 32
|
||||
LOW_BLOCK_N = 64
|
||||
LOW_NUM_WARPS = 4
|
||||
THROUGHPUT_BLOCK_M = 32
|
||||
THROUGHPUT_BLOCK_N = 128
|
||||
THROUGHPUT_GROUP_M = 2
|
||||
THROUGHPUT_NUM_WARPS = 4
|
||||
SMALL_TOKEN_THRESHOLD = 128
|
||||
SMALL_NUM_WARPS = 2
|
||||
Q_BLOCK_ROWS = 8
|
||||
Q_NUM_WARPS = 2
|
||||
KV_BLOCK_ROWS = 4
|
||||
KV_NUM_WARPS = 2
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["rows"])
|
||||
def _rel_proj_low_latency_kernel(
|
||||
qkvr_ptr,
|
||||
rel_proj_ptr,
|
||||
rel_out_ptr,
|
||||
log_scaling_ptr,
|
||||
rows,
|
||||
stride_x_t,
|
||||
R_OFFSET: tl.constexpr,
|
||||
NUM_Q_HEADS: tl.constexpr,
|
||||
REL_EXTENT: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
APPLY_LOG_SCALING: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
col = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
inner = tl.arange(0, 16)
|
||||
token = row // NUM_Q_HEADS
|
||||
head = row % NUM_Q_HEADS
|
||||
relative = tl.load(
|
||||
qkvr_ptr
|
||||
+ token[:, None] * stride_x_t
|
||||
+ R_OFFSET
|
||||
+ head[:, None] * 16
|
||||
+ inner[None, :],
|
||||
mask=row[:, None] < rows,
|
||||
other=0.0,
|
||||
)
|
||||
projection = tl.load(
|
||||
rel_proj_ptr + inner[:, None] * REL_EXTENT + col[None, :],
|
||||
mask=col[None, :] < REL_EXTENT,
|
||||
other=0.0,
|
||||
)
|
||||
values = tl.dot(relative, projection, out_dtype=tl.float32).to(
|
||||
rel_out_ptr.dtype.element_ty
|
||||
)
|
||||
values = values.to(tl.float32)
|
||||
if APPLY_LOG_SCALING:
|
||||
values *= tl.load(log_scaling_ptr + token, mask=row < rows, other=1.0)[:, None]
|
||||
tl.store(
|
||||
rel_out_ptr + row[:, None] * REL_EXTENT + col[None, :],
|
||||
values.to(rel_out_ptr.dtype.element_ty),
|
||||
mask=(row[:, None] < rows) & (col[None, :] < REL_EXTENT),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["rows"])
|
||||
def _rel_proj_throughput_kernel(
|
||||
qkvr_ptr,
|
||||
rel_proj_ptr,
|
||||
rel_out_ptr,
|
||||
log_scaling_ptr,
|
||||
rows,
|
||||
stride_x_t,
|
||||
R_OFFSET: tl.constexpr,
|
||||
NUM_Q_HEADS: tl.constexpr,
|
||||
REL_EXTENT: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
GROUP_M: tl.constexpr,
|
||||
APPLY_LOG_SCALING: tl.constexpr,
|
||||
):
|
||||
row_group = tl.program_id(0)
|
||||
col = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
inner = tl.arange(0, 16)
|
||||
projection = tl.load(
|
||||
rel_proj_ptr + inner[:, None] * REL_EXTENT + col[None, :],
|
||||
mask=col[None, :] < REL_EXTENT,
|
||||
other=0.0,
|
||||
)
|
||||
row_offsets = tl.arange(0, BLOCK_M)
|
||||
for group_offset in tl.static_range(GROUP_M):
|
||||
row = (row_group * GROUP_M + group_offset) * BLOCK_M + row_offsets
|
||||
token = row // NUM_Q_HEADS
|
||||
head = row % NUM_Q_HEADS
|
||||
relative = tl.load(
|
||||
qkvr_ptr
|
||||
+ token[:, None] * stride_x_t
|
||||
+ R_OFFSET
|
||||
+ head[:, None] * 16
|
||||
+ inner[None, :],
|
||||
mask=row[:, None] < rows,
|
||||
other=0.0,
|
||||
)
|
||||
values = tl.dot(relative, projection, out_dtype=tl.float32).to(
|
||||
rel_out_ptr.dtype.element_ty
|
||||
)
|
||||
values = values.to(tl.float32)
|
||||
if APPLY_LOG_SCALING:
|
||||
values *= tl.load(log_scaling_ptr + token, mask=row < rows, other=1.0)[
|
||||
:, None
|
||||
]
|
||||
tl.store(
|
||||
rel_out_ptr + row[:, None] * REL_EXTENT + col[None, :],
|
||||
values.to(rel_out_ptr.dtype.element_ty),
|
||||
mask=(row[:, None] < rows) & (col[None, :] < REL_EXTENT),
|
||||
)
|
||||
|
||||
|
||||
def use_rel_proj_throughput(rows: int, rel_extent: int) -> bool:
|
||||
min_rows = 8192 if rel_extent == 512 else 2048
|
||||
return rows >= min_rows
|
||||
|
||||
|
||||
def qkvr_rel_proj(
|
||||
qkvr: torch.Tensor,
|
||||
rel_proj: torch.Tensor,
|
||||
rel_out: torch.Tensor,
|
||||
log_scaling: torch.Tensor | None,
|
||||
*,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
d_rel: int,
|
||||
) -> None:
|
||||
rows = qkvr.shape[0] * num_q_heads
|
||||
rel_extent = rel_proj.shape[1]
|
||||
assert d_rel == 16 and rel_proj.shape[0] == 16
|
||||
r_offset = num_q_heads * head_dim + 2 * num_kv_heads * head_dim
|
||||
log_scaling_ptr = log_scaling if log_scaling is not None else qkvr
|
||||
common = dict(
|
||||
R_OFFSET=r_offset,
|
||||
NUM_Q_HEADS=num_q_heads,
|
||||
REL_EXTENT=rel_extent,
|
||||
APPLY_LOG_SCALING=log_scaling is not None,
|
||||
)
|
||||
|
||||
if use_rel_proj_throughput(rows, rel_extent):
|
||||
grid = (
|
||||
triton.cdiv(rows, THROUGHPUT_BLOCK_M * THROUGHPUT_GROUP_M),
|
||||
triton.cdiv(rel_extent, THROUGHPUT_BLOCK_N),
|
||||
)
|
||||
_rel_proj_throughput_kernel[grid](
|
||||
qkvr,
|
||||
rel_proj,
|
||||
rel_out,
|
||||
log_scaling_ptr,
|
||||
rows,
|
||||
qkvr.stride(0),
|
||||
BLOCK_M=THROUGHPUT_BLOCK_M,
|
||||
BLOCK_N=THROUGHPUT_BLOCK_N,
|
||||
GROUP_M=THROUGHPUT_GROUP_M,
|
||||
num_warps=THROUGHPUT_NUM_WARPS,
|
||||
**common,
|
||||
)
|
||||
return
|
||||
|
||||
grid = (
|
||||
triton.cdiv(rows, LOW_BLOCK_M),
|
||||
triton.cdiv(rel_extent, LOW_BLOCK_N),
|
||||
)
|
||||
_rel_proj_low_latency_kernel[grid](
|
||||
qkvr,
|
||||
rel_proj,
|
||||
rel_out,
|
||||
log_scaling_ptr,
|
||||
rows,
|
||||
qkvr.stride(0),
|
||||
BLOCK_M=LOW_BLOCK_M,
|
||||
BLOCK_N=LOW_BLOCK_N,
|
||||
num_warps=LOW_NUM_WARPS,
|
||||
**common,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["tokens", "stride_block_table_req", "max_blocks"])
|
||||
def _qkvr_qkv_kernel(
|
||||
qkvr_ptr,
|
||||
q_norm_weight_ptr,
|
||||
q_out_ptr,
|
||||
rel_proj_ptr,
|
||||
rel_out_ptr,
|
||||
k_weight_ptr,
|
||||
v_weight_ptr,
|
||||
k_norm_weight_ptr,
|
||||
conv_cache_ptr,
|
||||
key_cache_ptr,
|
||||
value_cache_ptr,
|
||||
positions_ptr,
|
||||
seq_idx_ptr,
|
||||
conv_slot_mapping_ptr,
|
||||
conv_block_table_ptr,
|
||||
query_start_ptr,
|
||||
attention_slot_mapping_ptr,
|
||||
log_scaling_ptr,
|
||||
tokens,
|
||||
eps,
|
||||
stride_x_t,
|
||||
stride_cc_block,
|
||||
stride_cc_head,
|
||||
stride_cc_token,
|
||||
stride_cc_dim,
|
||||
stride_kc_block,
|
||||
stride_kc_token,
|
||||
stride_kc_head,
|
||||
stride_vc_block,
|
||||
stride_vc_token,
|
||||
stride_vc_head,
|
||||
stride_block_table_req,
|
||||
max_blocks,
|
||||
conv_block_size,
|
||||
attention_page_size,
|
||||
Q_WIDTH: tl.constexpr,
|
||||
KV_WIDTH: tl.constexpr,
|
||||
NUM_Q_HEADS: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
OFF_K: tl.constexpr,
|
||||
OFF_V: tl.constexpr,
|
||||
APPLY_LOG_SCALING: tl.constexpr,
|
||||
D_REL: tl.constexpr,
|
||||
REL_EXTENT: tl.constexpr,
|
||||
REL_EXTENT_PADDED: tl.constexpr,
|
||||
):
|
||||
block = tl.program_id(0)
|
||||
num_q_rows = tokens * NUM_Q_HEADS
|
||||
dims = tl.arange(0, HEAD_DIM)
|
||||
|
||||
if block < num_q_rows:
|
||||
row = block
|
||||
token = row // NUM_Q_HEADS
|
||||
head = row % NUM_Q_HEADS
|
||||
values = tl.load(
|
||||
qkvr_ptr + token * stride_x_t + head * HEAD_DIM + dims,
|
||||
).to(tl.float32)
|
||||
weight = tl.load(q_norm_weight_ptr + dims).to(tl.float32)
|
||||
rstd = tl.rsqrt(tl.sum(values * values, axis=0) / HEAD_DIM + eps)
|
||||
normalized = values * rstd * weight
|
||||
if APPLY_LOG_SCALING:
|
||||
normalized = normalized.to(q_out_ptr.dtype.element_ty).to(tl.float32)
|
||||
normalized *= tl.load(log_scaling_ptr + token)
|
||||
tl.store(
|
||||
q_out_ptr + row * HEAD_DIM + dims,
|
||||
normalized.to(q_out_ptr.dtype.element_ty),
|
||||
)
|
||||
rel_cols = tl.arange(0, REL_EXTENT_PADDED)
|
||||
rel_mask = rel_cols < REL_EXTENT
|
||||
projected = tl.zeros([REL_EXTENT_PADDED], dtype=tl.float32)
|
||||
rel_offset = Q_WIDTH + 2 * KV_WIDTH + head * D_REL
|
||||
for rel_dim in tl.static_range(D_REL):
|
||||
rel_value = tl.load(
|
||||
qkvr_ptr + token * stride_x_t + rel_offset + rel_dim
|
||||
).to(tl.float32)
|
||||
proj = tl.load(
|
||||
rel_proj_ptr + rel_dim * REL_EXTENT + rel_cols,
|
||||
mask=rel_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
projected += rel_value * proj
|
||||
projected = projected.to(rel_out_ptr.dtype.element_ty).to(tl.float32)
|
||||
if APPLY_LOG_SCALING:
|
||||
projected *= tl.load(log_scaling_ptr + token)
|
||||
tl.store(
|
||||
rel_out_ptr + row * REL_EXTENT + rel_cols,
|
||||
projected.to(rel_out_ptr.dtype.element_ty),
|
||||
mask=rel_mask,
|
||||
)
|
||||
else:
|
||||
row = block - num_q_rows
|
||||
if row < tokens * NUM_KV_HEADS:
|
||||
token = row // NUM_KV_HEADS
|
||||
head = row % NUM_KV_HEADS
|
||||
position = tl.load(positions_ptr + token)
|
||||
request = tl.load(seq_idx_ptr + token)
|
||||
conv_slot = tl.load(conv_slot_mapping_ptr + token)
|
||||
query_start = tl.load(query_start_ptr + token)
|
||||
attention_slot = tl.load(attention_slot_mapping_ptr + token)
|
||||
valid = conv_slot >= 0
|
||||
|
||||
k_col = Q_WIDTH + head * HEAD_DIM
|
||||
v_col = Q_WIDTH + KV_WIDTH + head * HEAD_DIM
|
||||
k_value = tl.load(qkvr_ptr + token * stride_x_t + k_col + dims)
|
||||
v_value = tl.load(qkvr_ptr + token * stride_x_t + v_col + dims)
|
||||
|
||||
safe_slot = tl.maximum(conv_slot, 0)
|
||||
cache_base = (
|
||||
conv_cache_ptr
|
||||
+ (safe_slot // conv_block_size) * stride_cc_block
|
||||
+ head * stride_cc_head
|
||||
+ (safe_slot % conv_block_size) * stride_cc_token
|
||||
)
|
||||
tl.store(
|
||||
cache_base + (OFF_K + dims) * stride_cc_dim,
|
||||
k_value,
|
||||
mask=valid,
|
||||
)
|
||||
tl.store(
|
||||
cache_base + (OFF_V + dims) * stride_cc_dim,
|
||||
v_value,
|
||||
mask=valid,
|
||||
)
|
||||
|
||||
acc_k = tl.zeros([HEAD_DIM], dtype=tl.float32)
|
||||
acc_v = tl.zeros([HEAD_DIM], dtype=tl.float32)
|
||||
for tap in tl.static_range(WINDOW_SIZE):
|
||||
source_position = position - (WINDOW_SIZE - 1) + tap
|
||||
source_row = token - (WINDOW_SIZE - 1) + tap
|
||||
in_window = valid & (source_position >= 0)
|
||||
intra = in_window & (source_row >= query_start)
|
||||
cached = in_window & (source_row < query_start)
|
||||
safe_row = tl.maximum(source_row, 0)
|
||||
source_k = tl.load(
|
||||
qkvr_ptr + safe_row * stride_x_t + k_col + dims,
|
||||
mask=intra,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
source_v = tl.load(
|
||||
qkvr_ptr + safe_row * stride_x_t + v_col + dims,
|
||||
mask=intra,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
safe_position = tl.maximum(source_position, 0)
|
||||
logical_block = tl.minimum(
|
||||
safe_position // conv_block_size, max_blocks - 1
|
||||
)
|
||||
physical_block = tl.load(
|
||||
conv_block_table_ptr
|
||||
+ request * stride_block_table_req
|
||||
+ logical_block,
|
||||
mask=cached,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
tap_base = (
|
||||
conv_cache_ptr
|
||||
+ physical_block * stride_cc_block
|
||||
+ head * stride_cc_head
|
||||
+ (safe_position % conv_block_size) * stride_cc_token
|
||||
)
|
||||
source_k += tl.load(
|
||||
tap_base + (OFF_K + dims) * stride_cc_dim,
|
||||
mask=cached,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
source_v += tl.load(
|
||||
tap_base + (OFF_V + dims) * stride_cc_dim,
|
||||
mask=cached,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
k_weight = tl.load(
|
||||
k_weight_ptr + (head * HEAD_DIM + dims) * WINDOW_SIZE + tap
|
||||
).to(tl.float32)
|
||||
v_weight = tl.load(
|
||||
v_weight_ptr + (head * HEAD_DIM + dims) * WINDOW_SIZE + tap
|
||||
).to(tl.float32)
|
||||
acc_k += source_k * k_weight
|
||||
acc_v += source_v * v_weight
|
||||
|
||||
k_rounded = (acc_k + k_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
|
||||
v_rounded = (acc_v + v_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
|
||||
k_float = k_rounded.to(tl.float32)
|
||||
k_norm_weight = tl.load(k_norm_weight_ptr + dims).to(tl.float32)
|
||||
rstd = tl.rsqrt(tl.sum(k_float * k_float, axis=0) / HEAD_DIM + eps)
|
||||
k_normalized = (k_float * rstd * k_norm_weight).to(
|
||||
qkvr_ptr.dtype.element_ty
|
||||
)
|
||||
|
||||
safe_attention_slot = tl.maximum(attention_slot, 0)
|
||||
attention_block = safe_attention_slot // attention_page_size
|
||||
attention_offset = safe_attention_slot % attention_page_size
|
||||
tl.store(
|
||||
key_cache_ptr
|
||||
+ attention_block * stride_kc_block
|
||||
+ attention_offset * stride_kc_token
|
||||
+ head * stride_kc_head
|
||||
+ dims,
|
||||
k_normalized,
|
||||
mask=attention_slot >= 0,
|
||||
)
|
||||
tl.store(
|
||||
value_cache_ptr
|
||||
+ attention_block * stride_vc_block
|
||||
+ attention_offset * stride_vc_token
|
||||
+ head * stride_vc_head
|
||||
+ dims,
|
||||
v_rounded,
|
||||
mask=attention_slot >= 0,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_rows"])
|
||||
def _q_kernel(
|
||||
qkvr_ptr,
|
||||
q_norm_weight_ptr,
|
||||
q_out_ptr,
|
||||
log_scaling_ptr,
|
||||
num_rows,
|
||||
stride_x_t,
|
||||
eps,
|
||||
NUM_Q_HEADS: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
APPLY_LOG_SCALING: tl.constexpr,
|
||||
):
|
||||
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
|
||||
dims = tl.arange(0, HEAD_DIM)
|
||||
row_mask = rows < num_rows
|
||||
tokens = rows // NUM_Q_HEADS
|
||||
heads = rows % NUM_Q_HEADS
|
||||
values = tl.load(
|
||||
qkvr_ptr
|
||||
+ tokens[:, None] * stride_x_t
|
||||
+ heads[:, None] * HEAD_DIM
|
||||
+ dims[None, :],
|
||||
mask=row_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
weight = tl.load(q_norm_weight_ptr + dims).to(tl.float32)
|
||||
rstd = tl.rsqrt(tl.sum(values * values, axis=1) / HEAD_DIM + eps)
|
||||
normalized = values * rstd[:, None] * weight[None, :]
|
||||
if APPLY_LOG_SCALING:
|
||||
normalized = normalized.to(q_out_ptr.dtype.element_ty).to(tl.float32)
|
||||
tau = tl.load(log_scaling_ptr + tokens, mask=row_mask, other=1.0)
|
||||
normalized *= tau[:, None]
|
||||
tl.store(
|
||||
q_out_ptr + rows[:, None] * HEAD_DIM + dims[None, :],
|
||||
normalized.to(q_out_ptr.dtype.element_ty),
|
||||
mask=row_mask[:, None],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["tokens", "stride_block_table_req", "max_blocks"])
|
||||
def _kv_kernel(
|
||||
qkvr_ptr,
|
||||
k_weight_ptr,
|
||||
v_weight_ptr,
|
||||
k_norm_weight_ptr,
|
||||
conv_cache_ptr,
|
||||
key_cache_ptr,
|
||||
value_cache_ptr,
|
||||
positions_ptr,
|
||||
seq_idx_ptr,
|
||||
conv_slot_mapping_ptr,
|
||||
conv_block_table_ptr,
|
||||
query_start_ptr,
|
||||
attention_slot_mapping_ptr,
|
||||
tokens,
|
||||
eps,
|
||||
stride_x_t,
|
||||
stride_cc_block,
|
||||
stride_cc_head,
|
||||
stride_cc_token,
|
||||
stride_cc_dim,
|
||||
stride_kc_block,
|
||||
stride_kc_token,
|
||||
stride_kc_head,
|
||||
stride_vc_block,
|
||||
stride_vc_token,
|
||||
stride_vc_head,
|
||||
stride_block_table_req,
|
||||
max_blocks,
|
||||
conv_block_size,
|
||||
attention_page_size,
|
||||
Q_WIDTH: tl.constexpr,
|
||||
KV_WIDTH: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
OFF_K: tl.constexpr,
|
||||
OFF_V: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
):
|
||||
token = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
|
||||
dims = tl.arange(0, HEAD_DIM)
|
||||
row_mask = token < tokens
|
||||
head_id = tl.program_id(1)
|
||||
head = tl.full([BLOCK_ROWS], head_id, tl.int64)
|
||||
position = tl.load(positions_ptr + token, mask=row_mask, other=0)
|
||||
request = tl.load(seq_idx_ptr + token, mask=row_mask, other=0)
|
||||
conv_slot = tl.load(conv_slot_mapping_ptr + token, mask=row_mask, other=-1)
|
||||
query_start = tl.load(query_start_ptr + token, mask=row_mask, other=0)
|
||||
attention_slot = tl.load(
|
||||
attention_slot_mapping_ptr + token, mask=row_mask, other=-1
|
||||
)
|
||||
valid = row_mask & (conv_slot >= 0)
|
||||
|
||||
k_col = Q_WIDTH + head * HEAD_DIM
|
||||
v_col = Q_WIDTH + KV_WIDTH + head * HEAD_DIM
|
||||
k_value = tl.load(
|
||||
qkvr_ptr + token[:, None] * stride_x_t + k_col[:, None] + dims[None, :],
|
||||
mask=row_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
v_value = tl.load(
|
||||
qkvr_ptr + token[:, None] * stride_x_t + v_col[:, None] + dims[None, :],
|
||||
mask=row_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
safe_slot = tl.maximum(conv_slot, 0)
|
||||
cache_base = (
|
||||
conv_cache_ptr
|
||||
+ (safe_slot // conv_block_size) * stride_cc_block
|
||||
+ head * stride_cc_head
|
||||
+ (safe_slot % conv_block_size) * stride_cc_token
|
||||
)
|
||||
tl.store(
|
||||
cache_base[:, None] + (OFF_K + dims[None, :]) * stride_cc_dim,
|
||||
k_value,
|
||||
mask=valid[:, None],
|
||||
)
|
||||
tl.store(
|
||||
cache_base[:, None] + (OFF_V + dims[None, :]) * stride_cc_dim,
|
||||
v_value,
|
||||
mask=valid[:, None],
|
||||
)
|
||||
|
||||
acc_k = tl.zeros([BLOCK_ROWS, HEAD_DIM], dtype=tl.float32)
|
||||
acc_v = tl.zeros([BLOCK_ROWS, HEAD_DIM], dtype=tl.float32)
|
||||
for tap in tl.static_range(WINDOW_SIZE):
|
||||
source_position = position - (WINDOW_SIZE - 1) + tap
|
||||
source_row = token - (WINDOW_SIZE - 1) + tap
|
||||
in_window = valid & (source_position >= 0)
|
||||
intra = in_window & (source_row >= query_start)
|
||||
cached = in_window & (source_row < query_start)
|
||||
safe_row = tl.maximum(source_row, 0)
|
||||
source_k = tl.load(
|
||||
qkvr_ptr + safe_row[:, None] * stride_x_t + k_col[:, None] + dims[None, :],
|
||||
mask=intra[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
source_v = tl.load(
|
||||
qkvr_ptr + safe_row[:, None] * stride_x_t + v_col[:, None] + dims[None, :],
|
||||
mask=intra[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
safe_position = tl.maximum(source_position, 0)
|
||||
logical_block = tl.minimum(safe_position // conv_block_size, max_blocks - 1)
|
||||
physical_block = tl.load(
|
||||
conv_block_table_ptr + request * stride_block_table_req + logical_block,
|
||||
mask=cached,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
tap_base = (
|
||||
conv_cache_ptr
|
||||
+ physical_block * stride_cc_block
|
||||
+ head * stride_cc_head
|
||||
+ (safe_position % conv_block_size) * stride_cc_token
|
||||
)
|
||||
source_k += tl.load(
|
||||
tap_base[:, None] + (OFF_K + dims[None, :]) * stride_cc_dim,
|
||||
mask=cached[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
source_v += tl.load(
|
||||
tap_base[:, None] + (OFF_V + dims[None, :]) * stride_cc_dim,
|
||||
mask=cached[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
k_weight = tl.load(
|
||||
k_weight_ptr + (head_id * HEAD_DIM + dims) * WINDOW_SIZE + tap
|
||||
).to(tl.float32)
|
||||
v_weight = tl.load(
|
||||
v_weight_ptr + (head_id * HEAD_DIM + dims) * WINDOW_SIZE + tap
|
||||
).to(tl.float32)
|
||||
acc_k += source_k * k_weight[None, :]
|
||||
acc_v += source_v * v_weight[None, :]
|
||||
|
||||
k_rounded = (acc_k + k_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
|
||||
v_rounded = (acc_v + v_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
|
||||
k_float = k_rounded.to(tl.float32)
|
||||
k_norm_weight = tl.load(k_norm_weight_ptr + dims).to(tl.float32)
|
||||
rstd = tl.rsqrt(tl.sum(k_float * k_float, axis=1) / HEAD_DIM + eps)
|
||||
k_normalized = (k_float * rstd[:, None] * k_norm_weight[None, :]).to(
|
||||
qkvr_ptr.dtype.element_ty
|
||||
)
|
||||
|
||||
safe_attention_slot = tl.maximum(attention_slot, 0)
|
||||
attention_block = safe_attention_slot // attention_page_size
|
||||
attention_offset = safe_attention_slot % attention_page_size
|
||||
attention_mask = row_mask & (attention_slot >= 0)
|
||||
tl.store(
|
||||
key_cache_ptr
|
||||
+ attention_block[:, None] * stride_kc_block
|
||||
+ attention_offset[:, None] * stride_kc_token
|
||||
+ head[:, None] * stride_kc_head
|
||||
+ dims[None, :],
|
||||
k_normalized,
|
||||
mask=attention_mask[:, None],
|
||||
)
|
||||
tl.store(
|
||||
value_cache_ptr
|
||||
+ attention_block[:, None] * stride_vc_block
|
||||
+ attention_offset[:, None] * stride_vc_token
|
||||
+ head[:, None] * stride_vc_head
|
||||
+ dims[None, :],
|
||||
v_rounded,
|
||||
mask=attention_mask[:, None],
|
||||
)
|
||||
|
||||
|
||||
def _run_tiled_q(
|
||||
qkvr: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
*,
|
||||
eps: float,
|
||||
num_q_heads: int,
|
||||
head_dim: int,
|
||||
log_scaling: torch.Tensor | None,
|
||||
) -> None:
|
||||
num_rows = qkvr.shape[0] * num_q_heads
|
||||
_q_kernel[(triton.cdiv(num_rows, Q_BLOCK_ROWS),)](
|
||||
qkvr,
|
||||
q_norm_weight,
|
||||
q_out,
|
||||
log_scaling if log_scaling is not None else positions,
|
||||
num_rows,
|
||||
qkvr.stride(0),
|
||||
eps,
|
||||
NUM_Q_HEADS=num_q_heads,
|
||||
HEAD_DIM=head_dim,
|
||||
BLOCK_ROWS=Q_BLOCK_ROWS,
|
||||
APPLY_LOG_SCALING=log_scaling is not None,
|
||||
num_warps=Q_NUM_WARPS,
|
||||
)
|
||||
|
||||
|
||||
def _run_tiled_kv(
|
||||
qkvr: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
v_weight: torch.Tensor,
|
||||
k_norm_weight: torch.Tensor,
|
||||
conv_cache: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
seq_idx: torch.Tensor,
|
||||
conv_slot_mapping: torch.Tensor,
|
||||
conv_block_table: torch.Tensor,
|
||||
query_start: torch.Tensor,
|
||||
attention_slot_mapping: torch.Tensor,
|
||||
*,
|
||||
eps: float,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
off_k: int,
|
||||
off_v: int,
|
||||
conv_block_size: int,
|
||||
) -> None:
|
||||
tokens = qkvr.shape[0]
|
||||
_kv_kernel[(triton.cdiv(tokens, KV_BLOCK_ROWS), num_kv_heads)](
|
||||
qkvr,
|
||||
k_weight,
|
||||
v_weight,
|
||||
k_norm_weight,
|
||||
conv_cache,
|
||||
key_cache,
|
||||
value_cache,
|
||||
positions,
|
||||
seq_idx,
|
||||
conv_slot_mapping,
|
||||
conv_block_table,
|
||||
query_start,
|
||||
attention_slot_mapping,
|
||||
tokens,
|
||||
eps,
|
||||
qkvr.stride(0),
|
||||
conv_cache.stride(0),
|
||||
conv_cache.stride(1),
|
||||
conv_cache.stride(2),
|
||||
conv_cache.stride(3),
|
||||
key_cache.stride(0),
|
||||
key_cache.stride(1),
|
||||
key_cache.stride(2),
|
||||
value_cache.stride(0),
|
||||
value_cache.stride(1),
|
||||
value_cache.stride(2),
|
||||
conv_block_table.stride(0),
|
||||
conv_block_table.shape[1],
|
||||
conv_block_size,
|
||||
key_cache.shape[1],
|
||||
Q_WIDTH=num_q_heads * head_dim,
|
||||
KV_WIDTH=num_kv_heads * head_dim,
|
||||
NUM_KV_HEADS=num_kv_heads,
|
||||
HEAD_DIM=head_dim,
|
||||
WINDOW_SIZE=k_weight.shape[1],
|
||||
OFF_K=off_k,
|
||||
OFF_V=off_v,
|
||||
BLOCK_ROWS=KV_BLOCK_ROWS,
|
||||
num_warps=KV_NUM_WARPS,
|
||||
)
|
||||
|
||||
|
||||
def _run_fused_small(
|
||||
qkvr: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
rel_proj: torch.Tensor,
|
||||
rel_out: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
v_weight: torch.Tensor,
|
||||
k_norm_weight: torch.Tensor,
|
||||
conv_cache: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
seq_idx: torch.Tensor,
|
||||
conv_slot_mapping: torch.Tensor,
|
||||
conv_block_table: torch.Tensor,
|
||||
query_start: torch.Tensor,
|
||||
attention_slot_mapping: torch.Tensor,
|
||||
*,
|
||||
eps: float,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
off_k: int,
|
||||
off_v: int,
|
||||
conv_block_size: int,
|
||||
log_scaling: torch.Tensor | None,
|
||||
) -> None:
|
||||
tokens = qkvr.shape[0]
|
||||
|
||||
num_q_rows = tokens * num_q_heads
|
||||
grid = (num_q_rows + tokens * num_kv_heads,)
|
||||
_qkvr_qkv_kernel[grid](
|
||||
qkvr,
|
||||
q_norm_weight,
|
||||
q_out,
|
||||
rel_proj,
|
||||
rel_out,
|
||||
k_weight,
|
||||
v_weight,
|
||||
k_norm_weight,
|
||||
conv_cache,
|
||||
key_cache,
|
||||
value_cache,
|
||||
positions,
|
||||
seq_idx,
|
||||
conv_slot_mapping,
|
||||
conv_block_table,
|
||||
query_start,
|
||||
attention_slot_mapping,
|
||||
log_scaling if log_scaling is not None else positions,
|
||||
tokens,
|
||||
eps,
|
||||
qkvr.stride(0),
|
||||
conv_cache.stride(0),
|
||||
conv_cache.stride(1),
|
||||
conv_cache.stride(2),
|
||||
conv_cache.stride(3),
|
||||
key_cache.stride(0),
|
||||
key_cache.stride(1),
|
||||
key_cache.stride(2),
|
||||
value_cache.stride(0),
|
||||
value_cache.stride(1),
|
||||
value_cache.stride(2),
|
||||
conv_block_table.stride(0),
|
||||
conv_block_table.shape[1],
|
||||
conv_block_size,
|
||||
key_cache.shape[1],
|
||||
Q_WIDTH=num_q_heads * head_dim,
|
||||
KV_WIDTH=num_kv_heads * head_dim,
|
||||
NUM_Q_HEADS=num_q_heads,
|
||||
NUM_KV_HEADS=num_kv_heads,
|
||||
HEAD_DIM=head_dim,
|
||||
WINDOW_SIZE=k_weight.shape[1],
|
||||
OFF_K=off_k,
|
||||
OFF_V=off_v,
|
||||
APPLY_LOG_SCALING=log_scaling is not None,
|
||||
D_REL=16,
|
||||
REL_EXTENT=rel_proj.shape[1],
|
||||
REL_EXTENT_PADDED=triton.next_power_of_2(rel_proj.shape[1]),
|
||||
num_warps=SMALL_NUM_WARPS,
|
||||
)
|
||||
|
||||
|
||||
def fused_qkvr_prep(
|
||||
qkvr: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
v_weight: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
k_norm_weight: torch.Tensor,
|
||||
rel_proj: torch.Tensor,
|
||||
eps: float,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
d_rel: int,
|
||||
conv_cache: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
conv_block_table: torch.Tensor,
|
||||
seq_idx: torch.Tensor,
|
||||
conv_slot_mapping: torch.Tensor,
|
||||
query_start: torch.Tensor,
|
||||
attention_slot_mapping: torch.Tensor,
|
||||
off_k: int,
|
||||
off_v: int,
|
||||
conv_block_size: int,
|
||||
log_scaling: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert d_rel == 16 and rel_proj.shape[0] == 16
|
||||
assert head_dim == 128
|
||||
assert qkvr.is_contiguous()
|
||||
assert k_weight.stride() == (k_weight.shape[1], 1)
|
||||
assert v_weight.stride() == (v_weight.shape[1], 1)
|
||||
assert rel_proj.stride() == (rel_proj.shape[1], 1)
|
||||
assert conv_cache.stride(3) == 1
|
||||
assert key_cache.stride(3) == 1 and value_cache.stride(3) == 1
|
||||
tokens = qkvr.shape[0]
|
||||
q_out = torch.empty(
|
||||
(tokens, num_q_heads * head_dim), dtype=qkvr.dtype, device=qkvr.device
|
||||
)
|
||||
rel_out = torch.empty(
|
||||
(tokens, num_q_heads, rel_proj.shape[1]),
|
||||
dtype=qkvr.dtype,
|
||||
device=qkvr.device,
|
||||
)
|
||||
if tokens == 0:
|
||||
return q_out, rel_out
|
||||
|
||||
if tokens < SMALL_TOKEN_THRESHOLD:
|
||||
_run_fused_small(
|
||||
qkvr,
|
||||
q_norm_weight,
|
||||
q_out,
|
||||
rel_proj,
|
||||
rel_out,
|
||||
k_weight,
|
||||
v_weight,
|
||||
k_norm_weight,
|
||||
conv_cache,
|
||||
key_cache,
|
||||
value_cache,
|
||||
positions,
|
||||
seq_idx,
|
||||
conv_slot_mapping,
|
||||
conv_block_table,
|
||||
query_start,
|
||||
attention_slot_mapping,
|
||||
eps=eps,
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
off_k=off_k,
|
||||
off_v=off_v,
|
||||
conv_block_size=conv_block_size,
|
||||
log_scaling=log_scaling,
|
||||
)
|
||||
return q_out, rel_out
|
||||
|
||||
kv_stream = aux_stream()
|
||||
assert kv_stream is not None
|
||||
current_stream = torch.cuda.current_stream()
|
||||
kv_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(kv_stream):
|
||||
_run_tiled_kv(
|
||||
qkvr,
|
||||
k_weight,
|
||||
v_weight,
|
||||
k_norm_weight,
|
||||
conv_cache,
|
||||
key_cache,
|
||||
value_cache,
|
||||
positions,
|
||||
seq_idx,
|
||||
conv_slot_mapping,
|
||||
conv_block_table,
|
||||
query_start,
|
||||
attention_slot_mapping,
|
||||
eps=eps,
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
off_k=off_k,
|
||||
off_v=off_v,
|
||||
conv_block_size=conv_block_size,
|
||||
)
|
||||
_run_tiled_q(
|
||||
qkvr,
|
||||
q_norm_weight,
|
||||
q_out,
|
||||
positions,
|
||||
eps=eps,
|
||||
num_q_heads=num_q_heads,
|
||||
head_dim=head_dim,
|
||||
log_scaling=log_scaling,
|
||||
)
|
||||
qkvr_rel_proj(
|
||||
qkvr,
|
||||
rel_proj,
|
||||
rel_out,
|
||||
log_scaling,
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
d_rel=d_rel,
|
||||
)
|
||||
current_stream.wait_stream(kv_stream)
|
||||
return q_out, rel_out
|
||||
@@ -0,0 +1,403 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
"""Split-KV decode for Inkling relative attention on ROCm.
|
||||
|
||||
Adapted from LightSeek TokenSpeed's portable Triton relative-MHA decode.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_MIN_BLOCK_KV = tl.constexpr(32)
|
||||
|
||||
|
||||
def decode_split_count(max_kv_len: int, window_left: int) -> int:
|
||||
"""Return the number of parallel KV partitions for decode."""
|
||||
if window_left >= 0:
|
||||
effective_len = max(1, min(max_kv_len, window_left + 1))
|
||||
return min(4, max(1, triton.cdiv(effective_len, 128)))
|
||||
return min(32, max(1, triton.cdiv(max(1, max_kv_len), 2048)))
|
||||
|
||||
|
||||
def use_split_kv_decode(
|
||||
*,
|
||||
max_query_len: int,
|
||||
max_kv_len: int,
|
||||
page_size: int,
|
||||
window_left: int,
|
||||
) -> bool:
|
||||
"""Select split-KV only where it outperforms the single-pass kernel."""
|
||||
if os.getenv("INKLING_SPLIT_KV", "1") != "1":
|
||||
return False
|
||||
if max_query_len != 1:
|
||||
return False
|
||||
if page_size >= 64:
|
||||
return True
|
||||
if window_left >= 0:
|
||||
return page_size >= 32
|
||||
return max_kv_len >= 8192
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _split_kv_stage1(
|
||||
q_ptr,
|
||||
rel_ptr,
|
||||
k_ptr,
|
||||
v_ptr,
|
||||
block_table_ptr,
|
||||
cache_seqlens,
|
||||
mid_out_ptr,
|
||||
mid_lse_ptr,
|
||||
softmax_scale,
|
||||
stride_q_t,
|
||||
stride_q_h,
|
||||
stride_k_b,
|
||||
stride_k_p,
|
||||
stride_k_h,
|
||||
stride_k_d,
|
||||
stride_v_b,
|
||||
stride_v_p,
|
||||
stride_v_h,
|
||||
stride_v_d,
|
||||
stride_r_t,
|
||||
stride_r_h,
|
||||
stride_r_e,
|
||||
stride_mo_t,
|
||||
stride_mo_h,
|
||||
stride_mo_s,
|
||||
stride_ml_t,
|
||||
stride_ml_h,
|
||||
stride_ml_s,
|
||||
stride_bt_b: tl.constexpr,
|
||||
page_size: tl.constexpr,
|
||||
window_left: tl.constexpr,
|
||||
gqa_group_size: tl.constexpr,
|
||||
num_q_heads: tl.constexpr,
|
||||
head_dim: tl.constexpr,
|
||||
rel_extent: tl.constexpr,
|
||||
max_kv_splits: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
pid_t = tl.program_id(0)
|
||||
pid_h = tl.program_id(1)
|
||||
pid_s = tl.program_id(2)
|
||||
|
||||
head_blocks_per_kv = tl.cdiv(gqa_group_size, BLOCK_H)
|
||||
kv_head = pid_h // head_blocks_per_kv
|
||||
valid_block_h: tl.constexpr = min(BLOCK_H, gqa_group_size)
|
||||
off_h = pid_h * valid_block_h + tl.arange(0, BLOCK_H)
|
||||
h_valid = (off_h < (pid_h + 1) * valid_block_h) & (off_h < num_q_heads)
|
||||
off_d = tl.arange(0, BLOCK_D)
|
||||
d_valid = off_d < head_dim
|
||||
|
||||
cache_len = tl.load(cache_seqlens + pid_t)
|
||||
effective_len = (
|
||||
tl.minimum(cache_len, window_left + 1) if window_left >= 0 else cache_len
|
||||
)
|
||||
kv_offset = cache_len - effective_len
|
||||
split_len = (
|
||||
tl.cdiv(tl.cdiv(effective_len, max_kv_splits), _MIN_BLOCK_KV) * _MIN_BLOCK_KV
|
||||
)
|
||||
split_start = split_len * pid_s
|
||||
split_end = tl.minimum(split_start + split_len, effective_len)
|
||||
|
||||
q_offsets = pid_t * stride_q_t + off_h[:, None] * stride_q_h + off_d[None, :]
|
||||
row_max = tl.full((BLOCK_H,), float("-inf"), dtype=tl.float32)
|
||||
row_sum = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||||
acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32)
|
||||
|
||||
if split_end > split_start:
|
||||
q = tl.load(
|
||||
q_ptr + q_offsets,
|
||||
mask=h_valid[:, None] & d_valid[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
for start_n in range(split_start, split_end, BLOCK_N):
|
||||
off_n = start_n + tl.arange(0, BLOCK_N)
|
||||
n_valid = off_n < split_end
|
||||
token_idx = kv_offset + off_n
|
||||
logical_page = token_idx // page_size
|
||||
page_offset = token_idx % page_size
|
||||
physical_page = tl.load(
|
||||
block_table_ptr + pid_t * stride_bt_b + logical_page,
|
||||
mask=n_valid,
|
||||
other=0,
|
||||
)
|
||||
k_offsets = (
|
||||
physical_page[None, :].to(tl.int64) * stride_k_b
|
||||
+ page_offset[None, :] * stride_k_p
|
||||
+ kv_head * stride_k_h
|
||||
+ off_d[:, None] * stride_k_d
|
||||
)
|
||||
k = tl.load(
|
||||
k_ptr + k_offsets,
|
||||
mask=d_valid[:, None] & n_valid[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
scores = tl.dot(q, k.to(q.dtype)) * softmax_scale
|
||||
|
||||
rel_dist = cache_len - 1 - token_idx
|
||||
rel_valid = (rel_dist >= 0) & (rel_dist < rel_extent)
|
||||
rel_idx = tl.maximum(0, tl.minimum(rel_dist, rel_extent - 1))
|
||||
rel_offsets = (
|
||||
pid_t * stride_r_t
|
||||
+ off_h[:, None] * stride_r_h
|
||||
+ rel_idx[None, :] * stride_r_e
|
||||
)
|
||||
rel_bias = tl.load(
|
||||
rel_ptr + rel_offsets,
|
||||
mask=h_valid[:, None] & rel_valid[None, :] & n_valid[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
scores += rel_bias.to(tl.float32)
|
||||
scores = tl.where(
|
||||
h_valid[:, None] & n_valid[None, :],
|
||||
scores,
|
||||
float("-inf"),
|
||||
)
|
||||
|
||||
v_offsets = (
|
||||
physical_page[:, None].to(tl.int64) * stride_v_b
|
||||
+ page_offset[:, None] * stride_v_p
|
||||
+ kv_head * stride_v_h
|
||||
+ off_d[None, :] * stride_v_d
|
||||
)
|
||||
v = tl.load(
|
||||
v_ptr + v_offsets,
|
||||
mask=n_valid[:, None] & d_valid[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
next_max = tl.maximum(tl.max(scores, axis=1), row_max)
|
||||
old_scale = tl.exp(row_max - next_max)
|
||||
probs = tl.exp(scores - next_max[:, None])
|
||||
acc *= old_scale[:, None]
|
||||
acc += tl.dot(probs.to(v.dtype), v)
|
||||
row_sum = row_sum * old_scale + tl.sum(probs, axis=1)
|
||||
row_max = next_max
|
||||
|
||||
mid_out_offsets = (
|
||||
pid_t * stride_mo_t
|
||||
+ off_h[:, None] * stride_mo_h
|
||||
+ pid_s * stride_mo_s
|
||||
+ off_d[None, :]
|
||||
)
|
||||
tl.store(
|
||||
mid_out_ptr + mid_out_offsets,
|
||||
acc / row_sum[:, None],
|
||||
mask=h_valid[:, None] & d_valid[None, :],
|
||||
)
|
||||
mid_lse_offsets = (
|
||||
pid_t * stride_ml_t + off_h * stride_ml_h + pid_s * stride_ml_s
|
||||
)
|
||||
tl.store(
|
||||
mid_lse_ptr + mid_lse_offsets,
|
||||
row_max + tl.log(row_sum),
|
||||
mask=h_valid,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _split_kv_stage2(
|
||||
mid_out_ptr,
|
||||
mid_lse_ptr,
|
||||
out_ptr,
|
||||
cache_seqlens,
|
||||
stride_mo_t,
|
||||
stride_mo_h,
|
||||
stride_mo_s,
|
||||
stride_ml_t,
|
||||
stride_ml_h,
|
||||
stride_ml_s,
|
||||
stride_o_t,
|
||||
stride_o_h,
|
||||
window_left: tl.constexpr,
|
||||
head_dim: tl.constexpr,
|
||||
max_kv_splits: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
pid_t = tl.program_id(0)
|
||||
pid_h = tl.program_id(1)
|
||||
|
||||
cache_len = tl.load(cache_seqlens + pid_t)
|
||||
effective_len = (
|
||||
tl.minimum(cache_len, window_left + 1) if window_left >= 0 else cache_len
|
||||
)
|
||||
split_len = (
|
||||
tl.cdiv(tl.cdiv(effective_len, max_kv_splits), _MIN_BLOCK_KV) * _MIN_BLOCK_KV
|
||||
)
|
||||
|
||||
off_d = tl.arange(0, BLOCK_D)
|
||||
d_valid = off_d < head_dim
|
||||
row_max = -float("inf")
|
||||
row_sum = 0.0
|
||||
acc = tl.zeros((BLOCK_D,), dtype=tl.float32)
|
||||
|
||||
for split_id in range(max_kv_splits):
|
||||
split_start = split_len * split_id
|
||||
split_end = tl.minimum(split_start + split_len, effective_len)
|
||||
if split_end > split_start:
|
||||
value = tl.load(
|
||||
mid_out_ptr
|
||||
+ pid_t * stride_mo_t
|
||||
+ pid_h * stride_mo_h
|
||||
+ split_id * stride_mo_s
|
||||
+ off_d,
|
||||
mask=d_valid,
|
||||
other=0.0,
|
||||
)
|
||||
split_lse = tl.load(
|
||||
mid_lse_ptr
|
||||
+ pid_t * stride_ml_t
|
||||
+ pid_h * stride_ml_h
|
||||
+ split_id * stride_ml_s
|
||||
)
|
||||
next_max = tl.maximum(split_lse, row_max)
|
||||
old_scale = tl.exp(row_max - next_max)
|
||||
split_scale = tl.exp(split_lse - next_max)
|
||||
acc = acc * old_scale + value * split_scale
|
||||
row_sum = row_sum * old_scale + split_scale
|
||||
row_max = next_max
|
||||
|
||||
tl.store(
|
||||
out_ptr + pid_t * stride_o_t + pid_h * stride_o_h + off_d,
|
||||
acc / row_sum,
|
||||
mask=d_valid,
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def inkling_rel_attention_split_kv_decode(
|
||||
q: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
*,
|
||||
block_table: torch.Tensor,
|
||||
cache_seqlens: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
window_left: int,
|
||||
rel_extent: int,
|
||||
rel_logits: torch.Tensor,
|
||||
max_kv_len: int,
|
||||
out: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Run split-KV relative attention for single-token decode."""
|
||||
num_kv_heads = key_cache.shape[2]
|
||||
gqa_group_size = q.shape[1] // num_kv_heads
|
||||
max_kv_splits = decode_split_count(max_kv_len, window_left)
|
||||
block_d = triton.next_power_of_2(q.shape[2])
|
||||
block_h = min(8, gqa_group_size)
|
||||
|
||||
mid_out = torch.empty(
|
||||
q.shape[0],
|
||||
q.shape[1],
|
||||
max_kv_splits,
|
||||
q.shape[2],
|
||||
dtype=torch.float32,
|
||||
device=q.device,
|
||||
)
|
||||
mid_lse = torch.empty(
|
||||
q.shape[0],
|
||||
q.shape[1],
|
||||
max_kv_splits,
|
||||
dtype=torch.float32,
|
||||
device=q.device,
|
||||
)
|
||||
stage1_grid = (
|
||||
q.shape[0],
|
||||
triton.cdiv(q.shape[1], block_h),
|
||||
max_kv_splits,
|
||||
)
|
||||
_split_kv_stage1[stage1_grid](
|
||||
q,
|
||||
rel_logits,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table,
|
||||
cache_seqlens,
|
||||
mid_out,
|
||||
mid_lse,
|
||||
softmax_scale,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
key_cache.stride(0),
|
||||
key_cache.stride(1),
|
||||
key_cache.stride(2),
|
||||
key_cache.stride(3),
|
||||
value_cache.stride(0),
|
||||
value_cache.stride(1),
|
||||
value_cache.stride(2),
|
||||
value_cache.stride(3),
|
||||
rel_logits.stride(0),
|
||||
rel_logits.stride(1),
|
||||
rel_logits.stride(2),
|
||||
mid_out.stride(0),
|
||||
mid_out.stride(1),
|
||||
mid_out.stride(2),
|
||||
mid_lse.stride(0),
|
||||
mid_lse.stride(1),
|
||||
mid_lse.stride(2),
|
||||
block_table.stride(0),
|
||||
page_size=key_cache.shape[1],
|
||||
window_left=window_left,
|
||||
gqa_group_size=gqa_group_size,
|
||||
num_q_heads=q.shape[1],
|
||||
head_dim=q.shape[2],
|
||||
rel_extent=rel_extent,
|
||||
max_kv_splits=max_kv_splits,
|
||||
BLOCK_D=block_d,
|
||||
BLOCK_H=block_h,
|
||||
BLOCK_N=key_cache.shape[1],
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
)
|
||||
stage2_grid = (q.shape[0], q.shape[1])
|
||||
_split_kv_stage2[stage2_grid](
|
||||
mid_out,
|
||||
mid_lse,
|
||||
out,
|
||||
cache_seqlens,
|
||||
mid_out.stride(0),
|
||||
mid_out.stride(1),
|
||||
mid_out.stride(2),
|
||||
mid_lse.stride(0),
|
||||
mid_lse.stride(1),
|
||||
mid_lse.stride(2),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
window_left=window_left,
|
||||
head_dim=q.shape[2],
|
||||
max_kv_splits=max_kv_splits,
|
||||
BLOCK_D=block_d,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,290 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling short-convolution kernels backed by a paged sliding-window state cache.
|
||||
|
||||
Each layer's 4 conv streams (K, V, attn-output, mlp-output) share one paged KV
|
||||
cache ``[num_blocks, H, N, D]`` (head-major; see ``sconv_swa_attn.py``). A stream
|
||||
occupies the contiguous D-sub-range ``[off_s, off_s + ws)`` across all ``H``
|
||||
heads, so its flat per-token width is ``H * ws`` and that is the conv channel
|
||||
dim. The cache stores the conv *input* at every absolute position.
|
||||
|
||||
``fused_sconv`` is the single-launch path used by the model: per token it writes
|
||||
the current input to its slot and convolves the ``W`` taps ending at its
|
||||
absolute position. A tap landing inside the current forward is read from the
|
||||
immutable input ``x`` (row ``src - pos + pid_t``); only pre-forward taps are read
|
||||
from the paged cache (window position ``src`` -> physical block via
|
||||
``block_table[req, src // N]``). The just-written slot is never read back this
|
||||
step, so there is no write/read hazard within or across programs -- which is
|
||||
why this needs no decode-vs-prefill split and is valid for prefill / decode /
|
||||
spec alike.
|
||||
|
||||
All kernels address the cache purely by ``(slot, absolute_position)`` and
|
||||
allocate nothing inside the captured region; their grids depend only on the
|
||||
token count (``fused_sconv`` on a fixed token/channel tiling), so the same
|
||||
path replays correctly under eager, breakable PIECEWISE, and FULL cudagraphs
|
||||
without any data-dependent shape or branch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_sconv_kernel(
|
||||
x_ptr, # [T, H*WS] head-major current-token inputs (also residual)
|
||||
cache_ptr, # [num_blocks, H, N, D] paged (page-strided view)
|
||||
weight_ptr, # [H*WS, W]
|
||||
out_ptr, # [T, H*WS]
|
||||
pos_ptr, # [T] int64 absolute position per token
|
||||
seq_idx_ptr, # [T] int32 token -> batch request
|
||||
slot_ptr, # [T] int64 flat slot (block*N + blk_off); < 0 => PAD
|
||||
block_table_ptr, # [num_reqs, max_blocks] int32 block_table
|
||||
qstart_ptr, # [T] int32 first x-row of the token's request
|
||||
T, # num tokens
|
||||
stride_x_t,
|
||||
stride_c_blk,
|
||||
stride_c_h,
|
||||
stride_c_n,
|
||||
stride_c_d,
|
||||
stride_w_d,
|
||||
stride_w_w,
|
||||
stride_bt_r,
|
||||
MAX_BLOCKS,
|
||||
N, # block_size
|
||||
W: tl.constexpr,
|
||||
USE_SILU: tl.constexpr,
|
||||
USE_RESIDUAL: tl.constexpr,
|
||||
OFF_S: tl.constexpr,
|
||||
WS: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
BT: tl.constexpr,
|
||||
BLOCK_C: tl.constexpr,
|
||||
):
|
||||
# Each program owns a [BT tokens, BLOCK_C channels] tile. The flat channel
|
||||
# index packs all H heads head-major (head = c // WS, in-stream offset =
|
||||
# c % WS), so one program spans heads -- no per-head launch and no
|
||||
# next_power_of_2(WS) lane waste.
|
||||
pid_t = tl.program_id(0)
|
||||
pid_c = tl.program_id(1)
|
||||
toff = pid_t * BT + tl.arange(0, BT) # [BT] token rows
|
||||
coff = pid_c * BLOCK_C + tl.arange(0, BLOCK_C) # [BLOCK_C] flat channels
|
||||
C = H * WS
|
||||
t_mask = toff < T
|
||||
c_mask = coff < C
|
||||
head = tl.minimum(coff // WS, H - 1) # clamp keeps masked lanes in-buffer
|
||||
cd = OFF_S + coff % WS # cache D-index of the channel's stream slot
|
||||
|
||||
slot = tl.load(slot_ptr + toff, mask=t_mask, other=-1) # [BT]
|
||||
valid = slot >= 0
|
||||
pos = tl.load(pos_ptr + toff, mask=t_mask, other=0)
|
||||
req = tl.load(seq_idx_ptr + toff, mask=t_mask, other=0)
|
||||
qstart = tl.load(qstart_ptr + toff, mask=t_mask, other=0)
|
||||
|
||||
tc_mask = t_mask[:, None] & c_mask[None, :]
|
||||
|
||||
# 1) Insert each token's input into its paged slot (skip PAD rows).
|
||||
xv = tl.load(x_ptr + toff[:, None] * stride_x_t + coff[None, :], mask=tc_mask)
|
||||
safe_slot = tl.maximum(slot, 0)
|
||||
dst = (
|
||||
cache_ptr
|
||||
+ (safe_slot // N)[:, None] * stride_c_blk
|
||||
+ head[None, :] * stride_c_h
|
||||
+ (safe_slot % N)[:, None] * stride_c_n
|
||||
+ cd[None, :] * stride_c_d
|
||||
)
|
||||
tl.store(dst, xv, mask=tc_mask & valid[:, None])
|
||||
|
||||
# 2) Convolve the W taps ending at each token's `pos`. Each tap is read from
|
||||
# `x` if it falls inside this forward (row >= the request's first row), else
|
||||
# from the paged cache. Exactly one source is unmasked per tap, so we sum both.
|
||||
acc = tl.zeros([BT, BLOCK_C], dtype=tl.float32)
|
||||
for iw in tl.static_range(W):
|
||||
src = pos - (W - 1) + iw # [BT] absolute window position
|
||||
row = toff - (W - 1) + iw # [BT] x-row of `src` (== src - pos + token)
|
||||
in_win = valid & (src >= 0)
|
||||
intra = in_win & (row >= qstart)
|
||||
cached = in_win & (row < qstart)
|
||||
# intra-forward tap: read the immutable input x (never the slot we just
|
||||
# wrote), so there is no write/read hazard.
|
||||
safe_row = tl.maximum(row, 0)
|
||||
xt = tl.load(
|
||||
x_ptr + safe_row[:, None] * stride_x_t + coff[None, :],
|
||||
mask=c_mask[None, :] & intra[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
# pre-forward tap: read from the paged cache via the block table. Clamp
|
||||
# addressing terms; the load is masked off when out of window.
|
||||
safe_src = tl.maximum(src, 0)
|
||||
safe_lblk = tl.minimum(safe_src // N, MAX_BLOCKS - 1)
|
||||
blk = tl.load(
|
||||
block_table_ptr + req * stride_bt_r + safe_lblk, mask=cached, other=0
|
||||
).to(tl.int64)
|
||||
cbase = (
|
||||
cache_ptr
|
||||
+ blk[:, None] * stride_c_blk
|
||||
+ head[None, :] * stride_c_h
|
||||
+ (safe_src % N)[:, None] * stride_c_n
|
||||
+ cd[None, :] * stride_c_d
|
||||
)
|
||||
cv = tl.load(cbase, mask=c_mask[None, :] & cached[:, None], other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
wv = tl.load(
|
||||
weight_ptr + coff * stride_w_d + iw * stride_w_w, mask=c_mask, other=0.0
|
||||
).to(tl.float32)
|
||||
acc += (xt + cv) * wv[None, :]
|
||||
|
||||
if USE_SILU:
|
||||
acc = acc * tl.sigmoid(acc)
|
||||
if USE_RESIDUAL:
|
||||
acc += xv.to(tl.float32)
|
||||
|
||||
tl.store(
|
||||
out_ptr + toff[:, None] * stride_x_t + coff[None, :],
|
||||
acc.to(out_ptr.dtype.element_ty),
|
||||
mask=tc_mask,
|
||||
)
|
||||
|
||||
|
||||
def fused_sconv(
|
||||
x: torch.Tensor, # [T, H*ws] head-major current-token inputs
|
||||
weight: torch.Tensor, # [H*ws, W]
|
||||
cache: torch.Tensor, # [num_blocks, H, N, D] paged
|
||||
positions: torch.Tensor, # [T] int64 absolute position per token
|
||||
block_table: torch.Tensor, # [num_reqs, max_blocks] int32
|
||||
seq_idx: torch.Tensor, # [T] int32 token -> batch request
|
||||
slot_mapping: torch.Tensor, # [T] int64 flat slot (PAD = -1 => skip)
|
||||
query_start: torch.Tensor, # [T] int32 first x-row of the token's request
|
||||
off_s: int,
|
||||
ws: int,
|
||||
block_size: int,
|
||||
activation: str | None = None,
|
||||
use_residual: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Single-launch insert + depthwise causal conv1d over the paged cache.
|
||||
|
||||
Reads same-forward taps from ``x`` and pre-forward taps from the cache, so
|
||||
it is race-free in one launch for prefill / decode / spec and cudagraph-safe
|
||||
under eager / piecewise / full capture.
|
||||
"""
|
||||
T = x.shape[0]
|
||||
out = torch.empty_like(x)
|
||||
if T == 0:
|
||||
return out
|
||||
assert x.is_contiguous()
|
||||
assert cache.stride(3) == 1, "cache D-dim must be contiguous"
|
||||
H = cache.shape[1]
|
||||
W = weight.shape[1]
|
||||
C = H * ws # flat conv channel dim (all heads, head-major)
|
||||
# Tile BT tokens x BLOCK_C channels per program: enough work per CTA to
|
||||
# amortize the per-token addressing, while keeping the grid large for
|
||||
# prefill. BLOCK_C spans heads so there is no per-head launch. A ~2K-element
|
||||
# tile at 4 warps measured best on Blackwell; larger tiles spill registers.
|
||||
BLOCK_C = min(triton.next_power_of_2(C), 256)
|
||||
BT = 8
|
||||
grid = (triton.cdiv(T, BT), triton.cdiv(C, BLOCK_C))
|
||||
_fused_sconv_kernel[grid](
|
||||
x,
|
||||
cache,
|
||||
weight,
|
||||
out,
|
||||
positions,
|
||||
seq_idx,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
query_start,
|
||||
T,
|
||||
x.stride(0),
|
||||
cache.stride(0),
|
||||
cache.stride(1),
|
||||
cache.stride(2),
|
||||
cache.stride(3),
|
||||
weight.stride(0),
|
||||
weight.stride(1),
|
||||
block_table.stride(0),
|
||||
block_table.shape[1],
|
||||
block_size,
|
||||
W=W,
|
||||
USE_SILU=activation in ("silu", "swish"),
|
||||
USE_RESIDUAL=use_residual,
|
||||
OFF_S=off_s,
|
||||
WS=ws,
|
||||
H=H,
|
||||
BT=BT,
|
||||
BLOCK_C=BLOCK_C,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _seq_metadata_kernel(
|
||||
qsl_ptr, # [num_reqs + 1] int32 cumulative query start rows
|
||||
seq_idx_ptr, # [T] int32 out: token -> owning request
|
||||
query_start_ptr, # [T] int32 out: first x-row of the token's request
|
||||
num_reqs,
|
||||
num_actual_tokens,
|
||||
num_padded_tokens,
|
||||
n_iters, # ceil(log2(num_reqs)): binary-search depth
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
tok = offs.to(tl.int32)
|
||||
# Largest j in [0, num_reqs) with qsl[j] <= tok.
|
||||
lo = tl.zeros([BLOCK], tl.int32)
|
||||
hi = tl.full([BLOCK], num_reqs - 1, tl.int32)
|
||||
for _ in range(n_iters):
|
||||
mid = (lo + hi + 1) // 2
|
||||
below = tl.load(qsl_ptr + mid) <= tok
|
||||
lo = tl.where(below, mid, lo)
|
||||
hi = tl.where(below, hi, mid - 1)
|
||||
actual = offs < num_actual_tokens
|
||||
padded = offs < num_padded_tokens
|
||||
query_start = tl.load(qsl_ptr + lo)
|
||||
tl.store(seq_idx_ptr + offs, tl.where(actual, lo, 0), mask=padded)
|
||||
tl.store(
|
||||
query_start_ptr + offs,
|
||||
tl.where(actual, query_start, 0),
|
||||
mask=padded,
|
||||
)
|
||||
|
||||
|
||||
def sconv_seq_metadata(
|
||||
query_start_loc: torch.Tensor,
|
||||
num_reqs: int,
|
||||
num_actual_tokens: int,
|
||||
seq_idx_out: torch.Tensor,
|
||||
query_start_out: torch.Tensor,
|
||||
num_padded_tokens: int | None = None,
|
||||
) -> None:
|
||||
"""Fill static per-token seq_idx / query_start buffers in one launch.
|
||||
|
||||
Replaces the arange + searchsorted + clamp + gather + 2x copy chain of the
|
||||
sconv metadata build with a single kernel writing both persistent buffers.
|
||||
Padded rows are filled with zero and must have ``slot_mapping == -1``.
|
||||
"""
|
||||
if num_padded_tokens is None:
|
||||
num_padded_tokens = num_actual_tokens
|
||||
if num_padded_tokens < num_actual_tokens:
|
||||
raise ValueError("num_padded_tokens must cover all actual tokens")
|
||||
if num_padded_tokens > seq_idx_out.shape[0]:
|
||||
raise ValueError("seq_idx_out is too small for the padded token count")
|
||||
if num_padded_tokens > query_start_out.shape[0]:
|
||||
raise ValueError("query_start_out is too small for the padded token count")
|
||||
|
||||
BLOCK = 256
|
||||
n_iters = (num_reqs - 1).bit_length()
|
||||
grid = (triton.cdiv(num_padded_tokens, BLOCK),)
|
||||
_seq_metadata_kernel[grid](
|
||||
query_start_loc,
|
||||
seq_idx_out,
|
||||
query_start_out,
|
||||
num_reqs,
|
||||
num_actual_tokens,
|
||||
num_padded_tokens,
|
||||
n_iters,
|
||||
BLOCK=BLOCK,
|
||||
)
|
||||
@@ -0,0 +1,197 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""SwiGLU kernels for the Inkling MLP layers.
|
||||
|
||||
``silu_and_mul_triton``: SiLU-and-mul over the checkpoint's interleaved
|
||||
fused gate/up layout (dense MLP). ``sink_silu_mul_epilogue``: the sink-expert
|
||||
variant with the per-expert dequant scale and per-token gamma fused in.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["M"])
|
||||
def _silu_and_mul_triton_kernel(
|
||||
gateup_out_ptr,
|
||||
down_inp_ptr,
|
||||
M,
|
||||
N: tl.constexpr,
|
||||
GRID_SIZE: tl.constexpr,
|
||||
NUM_STAGES: tl.constexpr,
|
||||
BLOCK_SIZE_M: tl.constexpr,
|
||||
BLOCK_SIZE_N: tl.constexpr,
|
||||
EVEN_N: tl.constexpr,
|
||||
INT64_INDEX: tl.constexpr,
|
||||
):
|
||||
start_pid = tl.program_id(0)
|
||||
if INT64_INDEX:
|
||||
start_pid = start_pid.to(tl.int64)
|
||||
M = M.to(tl.int64)
|
||||
|
||||
NUM_BLOCKS_N: tl.constexpr = tl.cdiv(N, BLOCK_SIZE_N)
|
||||
num_blocks_mn = tl.cdiv(M, BLOCK_SIZE_M) * NUM_BLOCKS_N
|
||||
|
||||
for pid in tl.range(start_pid, num_blocks_mn, GRID_SIZE, num_stages=NUM_STAGES):
|
||||
pid_m = pid // NUM_BLOCKS_N
|
||||
pid_n = pid % NUM_BLOCKS_N
|
||||
|
||||
offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
|
||||
offs_n = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
|
||||
mask_m = offs_m < M
|
||||
mask_n = offs_n < N
|
||||
|
||||
# Interleaved fused gate/up: [g0, u0, g1, u1, ...].
|
||||
mask_offs_2n = pid_n * BLOCK_SIZE_N + tl.arange(0, 2 * BLOCK_SIZE_N) // 2
|
||||
tl.static_assert(BLOCK_SIZE_N % 8 == 0, f"{BLOCK_SIZE_N=}")
|
||||
mask_2n = mask_offs_2n < N
|
||||
mask_2n = tl.max_constancy(mask_2n, [16])
|
||||
|
||||
offs_2n = pid_n * 2 * BLOCK_SIZE_N + tl.arange(0, 2 * BLOCK_SIZE_N)
|
||||
offs_m2n = offs_m[:, None] * N * 2 + offs_2n[None, :]
|
||||
|
||||
if EVEN_N or pid_n * BLOCK_SIZE_N + BLOCK_SIZE_N <= N:
|
||||
gateup_out = tl.load(
|
||||
gateup_out_ptr + offs_m2n, mask=mask_m[:, None], other=0.0
|
||||
)
|
||||
else:
|
||||
mask_m2n = mask_m[:, None] & mask_2n[None, :]
|
||||
gateup_out = tl.load(gateup_out_ptr + offs_m2n, mask=mask_m2n, other=0.0)
|
||||
|
||||
gate_out, up_out = tl.split(
|
||||
tl.reshape(gateup_out, (BLOCK_SIZE_M, BLOCK_SIZE_N, 2))
|
||||
)
|
||||
gate_out = gate_out.to(tl.float32)
|
||||
up_out = up_out.to(tl.float32)
|
||||
|
||||
down_inp = gate_out * tl.sigmoid(gate_out) * up_out
|
||||
|
||||
mask_mn = mask_m[:, None] if EVEN_N else mask_m[:, None] & mask_n[None, :]
|
||||
offs_mn = offs_m[:, None] * N + offs_n[None, :]
|
||||
tl.store(down_inp_ptr + offs_mn, down_inp, mask=mask_mn)
|
||||
|
||||
|
||||
def silu_and_mul_triton(gateup_output: torch.Tensor) -> torch.Tensor:
|
||||
"""SiLU-and-mul for the interleaved fused gate/up layout.
|
||||
|
||||
Adapted from ``inkling_kernels.activation.silu_and_mul_fwd`` (without MXFP).
|
||||
"""
|
||||
assert gateup_output.is_contiguous(), (
|
||||
f"{gateup_output.shape=} {gateup_output.stride()=}"
|
||||
)
|
||||
assert gateup_output.ndim == 2, f"{gateup_output.shape=}"
|
||||
|
||||
M = gateup_output.shape[0]
|
||||
hidden_size = gateup_output.shape[1]
|
||||
assert hidden_size % 2 == 0, f"{hidden_size=}"
|
||||
N = hidden_size // 2
|
||||
|
||||
down_input = torch.empty(
|
||||
(M, N), device=gateup_output.device, dtype=gateup_output.dtype
|
||||
)
|
||||
if M == 0:
|
||||
return down_input
|
||||
|
||||
BLOCK_SIZE_N = max(8, min(256, triton.next_power_of_2(N)))
|
||||
if M <= 1:
|
||||
BLOCK_SIZE_M = 4
|
||||
elif M <= 256:
|
||||
BLOCK_SIZE_M = 2
|
||||
elif M < 4096:
|
||||
BLOCK_SIZE_M = 4
|
||||
else:
|
||||
BLOCK_SIZE_M = 16
|
||||
BLOCK_SIZE_N = max(8, min(128, triton.next_power_of_2(N)))
|
||||
max_grid_size = triton.cdiv(M, BLOCK_SIZE_M) * triton.cdiv(N, BLOCK_SIZE_N)
|
||||
num_sms = torch.cuda.get_device_properties(
|
||||
gateup_output.device
|
||||
).multi_processor_count
|
||||
grid_size = min(num_sms * 4, max_grid_size)
|
||||
|
||||
_silu_and_mul_triton_kernel[(grid_size,)](
|
||||
gateup_out_ptr=gateup_output,
|
||||
down_inp_ptr=down_input,
|
||||
M=M,
|
||||
N=N,
|
||||
GRID_SIZE=grid_size,
|
||||
NUM_STAGES=1,
|
||||
BLOCK_SIZE_M=BLOCK_SIZE_M,
|
||||
BLOCK_SIZE_N=BLOCK_SIZE_N,
|
||||
EVEN_N=N % BLOCK_SIZE_N == 0,
|
||||
INT64_INDEX=gateup_output.nbytes >= 2**31,
|
||||
num_warps=8,
|
||||
)
|
||||
|
||||
return down_input
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["T"])
|
||||
def _sink_epilogue_kernel(
|
||||
raw_ptr, # [T, S * 2F] gemm1 output, interleaved g/u pairs per expert block
|
||||
alpha_ptr, # [S] fp32 per-expert pre-SiLU dequant scale
|
||||
gamma_ptr, # [T, S] fp32 per-token sink weights (may be strided)
|
||||
ratio_ptr, # [S] fp32 per-expert post-SiLU scale (gemm2 alpha ratio)
|
||||
out_ptr, # [T, S * F] output
|
||||
T,
|
||||
stride_raw_0,
|
||||
stride_gamma_0,
|
||||
F: tl.constexpr,
|
||||
S: tl.constexpr,
|
||||
BLOCK_F: tl.constexpr,
|
||||
):
|
||||
pid_t = tl.program_id(0).to(tl.int64)
|
||||
pid_sf = tl.program_id(1)
|
||||
if pid_t >= T:
|
||||
return
|
||||
s = pid_sf // (F // BLOCK_F)
|
||||
offs_f = (pid_sf % (F // BLOCK_F)) * BLOCK_F + tl.arange(0, BLOCK_F)
|
||||
|
||||
base = pid_t * stride_raw_0 + s * 2 * F
|
||||
gate = tl.load(raw_ptr + base + 2 * offs_f).to(tl.float32)
|
||||
up = tl.load(raw_ptr + base + 2 * offs_f + 1).to(tl.float32)
|
||||
alpha = tl.load(alpha_ptr + s)
|
||||
weight = tl.load(gamma_ptr + pid_t * stride_gamma_0 + s) * tl.load(ratio_ptr + s)
|
||||
|
||||
gate *= alpha
|
||||
up *= alpha
|
||||
h = gate * tl.sigmoid(gate) * up * weight
|
||||
tl.store(out_ptr + pid_t * (S * F) + s * F + offs_f, h)
|
||||
|
||||
|
||||
def sink_silu_mul_epilogue(
|
||||
raw: torch.Tensor, # [T, S * 2F] gemm1 output (interleaved gate/up rows)
|
||||
alphas: torch.Tensor, # [S] fp32
|
||||
gammas: torch.Tensor, # [T, S] fp32
|
||||
ratios: torch.Tensor, # [S] fp32
|
||||
n_experts: int,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Fused sink-expert epilogue: silu(g * a_e) * (u * a_e) * (gamma * r_e).
|
||||
|
||||
One kernel replaces the per-expert dequant column scale, the SwiGLU, and
|
||||
the per-token gamma multiply between the two sink GEMMs.
|
||||
"""
|
||||
tokens = raw.shape[0]
|
||||
f = raw.shape[1] // (2 * n_experts)
|
||||
out = torch.empty((tokens, n_experts * f), device=raw.device, dtype=out_dtype)
|
||||
if tokens == 0:
|
||||
return out
|
||||
# raw may be a column-slice of a padded GEMM output (rows strided).
|
||||
assert raw.stride(1) == 1 and gammas.stride(1) == 1
|
||||
# Largest power-of-two divisor of f (f = 768 -> 256), capped at 512.
|
||||
block_f = min(512, f & (-f))
|
||||
_sink_epilogue_kernel[(tokens, n_experts * (f // block_f))](
|
||||
raw,
|
||||
alphas,
|
||||
gammas,
|
||||
ratios,
|
||||
out,
|
||||
tokens,
|
||||
raw.stride(0),
|
||||
gammas.stride(0),
|
||||
F=f,
|
||||
S=n_experts,
|
||||
BLOCK_F=block_f,
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,226 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling short-conv state managed as a sliding-window KV cache.
|
||||
|
||||
Each decoder layer owns one ``InklingConvState`` (an ``AttentionLayerBase``) that
|
||||
emits a single ``SlidingWindowSpec`` for the layer's 4 sconv streams (K, V,
|
||||
attn-output, mlp-output), packed head-major into one block:
|
||||
|
||||
H = num_kv_heads (per-rank), N = block_size = sconv_kernel_size,
|
||||
D = head_dim(K) + head_dim(V) + hidden/H(attn) + hidden/H(mlp)
|
||||
|
||||
``D`` is TP-invariant; per rank we store ``H/TP`` heads of width ``D``. The conv
|
||||
reads/writes this cache out-of-band via a custom backend; the (smaller) conv page
|
||||
is padded up to the uniform attention page by ``unify_kv_cache_spec_page_size``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionMetadata,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, SlidingWindowSpec
|
||||
|
||||
from .ops.sconv import sconv_seq_metadata
|
||||
|
||||
# Stream order within the per-head packed D (== contiguous sub-ranges).
|
||||
_K, _V, _ATTN, _MLP = 0, 1, 2, 3
|
||||
|
||||
|
||||
@dataclass
|
||||
class InklingSconvMetadata(AttentionMetadata):
|
||||
block_table: torch.Tensor # [num_reqs, max_blocks] physical blocks per req
|
||||
slot_mapping: torch.Tensor # [T] int64 flat slot of each token (-1 => skip)
|
||||
seq_idx: torch.Tensor # [T] int32 token -> batch request
|
||||
query_start: torch.Tensor # [T] int32 first x-row of each token's request
|
||||
|
||||
|
||||
class InklingSconvMetadataBuilder(AttentionMetadataBuilder[InklingSconvMetadata]):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
assert isinstance(kv_cache_spec, SlidingWindowSpec)
|
||||
# Persistent per-token buffers for CUDA graph capture.
|
||||
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.seq_idx_buffer = torch.empty(
|
||||
max_num_tokens, dtype=torch.int32, device=device
|
||||
)
|
||||
self.query_start_buffer = torch.empty(
|
||||
max_num_tokens, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> InklingSconvMetadata:
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
num_actual_tokens = int(common_attn_metadata.query_start_loc_cpu[-1])
|
||||
num_padded_tokens = common_attn_metadata.slot_mapping.shape[0]
|
||||
assert num_padded_tokens >= num_actual_tokens
|
||||
|
||||
# Per-token seq_idx (owning request) and query_start (first x-row of
|
||||
# that request; the fused kernel uses it to tell same-forward taps,
|
||||
# read from x, from pre-forward taps, read from cache) in one launch.
|
||||
sconv_seq_metadata(
|
||||
common_attn_metadata.query_start_loc,
|
||||
num_reqs,
|
||||
num_actual_tokens,
|
||||
self.seq_idx_buffer,
|
||||
self.query_start_buffer,
|
||||
num_padded_tokens,
|
||||
)
|
||||
|
||||
return InklingSconvMetadata(
|
||||
block_table=common_attn_metadata.block_table_tensor,
|
||||
slot_mapping=common_attn_metadata.slot_mapping[:num_padded_tokens],
|
||||
seq_idx=self.seq_idx_buffer[:num_padded_tokens],
|
||||
query_start=self.query_start_buffer[:num_padded_tokens],
|
||||
)
|
||||
|
||||
|
||||
class InklingSconvBackend(AttentionBackend):
|
||||
"""Custom dummy backend for the sconv sliding-window cache management."""
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "INKLING_SCONV_SWA"
|
||||
|
||||
@classmethod
|
||||
def indexes_kv_by_block_stride(cls) -> bool:
|
||||
# num_blocks is the outermost dim (HND, see get_kv_cache_shape), so the
|
||||
# padded conv page is read through a strided view.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
# HND, num-blocks-first, head-major: [num_blocks, H, N, D].
|
||||
return (num_blocks, num_kv_heads, block_size, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# Identity: physical layout == logical [num_blocks, H, N, D].
|
||||
if include_num_layers_dimension:
|
||||
return (0, 1, 2, 3, 4)
|
||||
return (0, 1, 2, 3)
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls():
|
||||
raise NotImplementedError(
|
||||
"InklingSconvBackend has no attention impl; the conv runs out-of-band."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[InklingSconvMetadataBuilder]:
|
||||
return InklingSconvMetadataBuilder
|
||||
|
||||
|
||||
class InklingConvState(nn.Module, AttentionLayerBase):
|
||||
"""Per-decoder-layer owner emitting one sliding-window conv-state spec."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
hidden_size: int,
|
||||
kernel_size: int,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.prefix = prefix
|
||||
# Bound to the manager-allocated paged cache by bind_kv_cache; a
|
||||
# placeholder until then. Read out-of-band by InklingShortConv.
|
||||
self.kv_cache = torch.tensor([])
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
# Guardrails for the conv-state layout below; only these are exercised.
|
||||
# tp_size <= num_kv_heads keeps >=1 whole KV head per rank (no
|
||||
# replication/clamping), so the per-head width stays TP-invariant.
|
||||
assert tp_size <= num_kv_heads, (
|
||||
f"sconv SWA cache supports tp_size <= num_kv_heads ({num_kv_heads}), "
|
||||
f"got {tp_size}"
|
||||
)
|
||||
# Per-rank head count; D is TP-invariant (K/V heads and the hidden
|
||||
# chunk both scale 1/TP together). The attn-/mlp-output sconv streams
|
||||
# are hidden-sharded: each rank owns its H/tp chunk (the sublayer
|
||||
# outputs are reduce-scattered / all-gathered around the conv).
|
||||
self.num_kv_heads = num_kv_heads // tp_size
|
||||
hidden_per_head = hidden_size // num_kv_heads
|
||||
# Packed per-head width: K + V + attn-output chunk + mlp-output chunk,
|
||||
# padded to a power of two so every layer's conv page is the same size
|
||||
# and an exact multiple of the attention page (the page unifier then
|
||||
# scales attention block sizes instead of padding).
|
||||
raw_head_size = 2 * head_dim + 2 * hidden_per_head
|
||||
self.head_size = 1 << (raw_head_size - 1).bit_length()
|
||||
self.sliding_window = kernel_size
|
||||
self.block_size = kernel_size
|
||||
# Per-head D-sub-range (offset, width) for each stream. Streams share
|
||||
# the cache; each writes/reads its own width across all H heads.
|
||||
self.stream_ranges: tuple[tuple[int, int], ...] = (
|
||||
(0, head_dim), # _K
|
||||
(head_dim, head_dim), # _V
|
||||
(2 * head_dim, hidden_per_head), # _ATTN
|
||||
(2 * head_dim + hidden_per_head, hidden_per_head), # _MLP
|
||||
)
|
||||
vllm_config = get_current_vllm_config()
|
||||
self._dtype = vllm_config.model_config.dtype
|
||||
assert self._dtype == torch.bfloat16, (
|
||||
f"sconv SWA cache supports bfloat16 only, got {self._dtype}"
|
||||
)
|
||||
# Register in the forward context so the runner enumerates this owner as
|
||||
# an attention-like layer (get_kv_cache_spec / get_attn_backend).
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def forward(self): ...
|
||||
|
||||
@property
|
||||
def cache_block_size(self) -> int:
|
||||
"""Return the block size used by cache metadata and physical indexing."""
|
||||
if self.kv_cache.numel() > 0:
|
||||
return self.kv_cache.shape[2]
|
||||
return self.block_size
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return InklingSconvBackend
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
return SlidingWindowSpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=0, # all 4 streams packed into head_size
|
||||
dtype=self._dtype,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
@@ -0,0 +1,98 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling short convolution: depthwise causal conv1d (+ residual) over a paged
|
||||
sliding-window conv-state cache.
|
||||
|
||||
Each decoder layer owns one ``InklingConvState`` (``sconv_swa_attn.py``) holding the
|
||||
manager-allocated paged cache for the layer's 4 sconv streams (K, V, attn-output,
|
||||
mlp-output), packed head-major into one block. Each ``InklingShortConv`` is a
|
||||
stateless weight + kernel launcher that, per forward (positions-addressed, the
|
||||
same path for prefill / decode / mixed), inserts the current tokens' inputs
|
||||
into their paged slot and convolves each token against the ``W`` taps ending
|
||||
at its absolute position, reading pre-forward window positions out of the
|
||||
paged cache via the block table.
|
||||
|
||||
Per-forward metadata (``block_table`` / ``slot_mapping`` / ``seq_idx`` /
|
||||
``query_start``) is built once by ``InklingSconvMetadataBuilder`` and published under
|
||||
the owner's prefix in the forward context; the absolute ``positions`` are
|
||||
threaded in from the model. The insert + conv run in a single ``fused_sconv``
|
||||
launch (same path for prefill / decode / mixed / spec). All inputs are
|
||||
fixed-address persistent buffers and the grid is fixed, so the conv replays
|
||||
correctly under eager, PIECEWISE, and FULL cudagraphs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.distributed import get_tensor_model_parallel_rank
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
from .ops import fused_sconv
|
||||
from .sconv_swa_attn import InklingConvState, InklingSconvMetadata
|
||||
|
||||
|
||||
class InklingShortConv(nn.Module):
|
||||
def __init__(
|
||||
self, dim: int, kernel_size: int, owner: InklingConvState, stream_idx: int
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.kernel_size = kernel_size
|
||||
self.owner = owner
|
||||
self.stream_idx = stream_idx
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
|
||||
# Depthwise conv weight; checkpoint stores (dim, 1, W).
|
||||
self.weight = Parameter(torch.empty(dim, 1, kernel_size), requires_grad=False)
|
||||
set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
|
||||
|
||||
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None:
|
||||
if loaded_weight.shape[0] != param.shape[0]:
|
||||
shard = param.shape[0]
|
||||
loaded_weight = loaded_weight.narrow(0, self.tp_rank * shard, shard)
|
||||
param.data.copy_(loaded_weight)
|
||||
|
||||
def forward(self, x: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
# x: (num_tokens, dim); positions: (num_tokens,) absolute positions.
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
# Memory-profiling / no metadata: identity (residual).
|
||||
return x
|
||||
m = attn_metadata.get(self.owner.prefix)
|
||||
if m is None:
|
||||
return x
|
||||
assert isinstance(m, InklingSconvMetadata)
|
||||
cache = self.owner.kv_cache
|
||||
if cache.numel() == 0:
|
||||
# Cache not yet bound (profiling before KV alloc): identity.
|
||||
return x
|
||||
|
||||
off_s, ws = self.owner.stream_ranges[self.stream_idx]
|
||||
# The hybrid KV-cache planner can enlarge the logical conv block so
|
||||
# that its physical page size matches the attention caches (for
|
||||
# example, W=4 becomes 32 when --block-size=128). Metadata slot
|
||||
# mappings are built with that enlarged size, so index the bound cache
|
||||
# with its runtime token dimension rather than the kernel window size.
|
||||
block_size = self.owner.cache_block_size
|
||||
x = x.contiguous()
|
||||
weight = self.weight.squeeze(1) # (dim, W)
|
||||
|
||||
return fused_sconv(
|
||||
x,
|
||||
weight,
|
||||
cache,
|
||||
positions,
|
||||
m.block_table,
|
||||
m.seq_idx,
|
||||
m.slot_mapping,
|
||||
m.query_start,
|
||||
off_s,
|
||||
ws,
|
||||
block_size,
|
||||
activation=None,
|
||||
use_residual=True,
|
||||
)
|
||||
@@ -12,10 +12,16 @@ if TYPE_CHECKING or HAS_TRITON:
|
||||
import triton
|
||||
import triton.language as tl
|
||||
import triton.language.extra.libdevice as tldevice
|
||||
from triton.experimental import gluon
|
||||
from triton.experimental.gluon import language as gl
|
||||
from triton.language.core import _aggregate as aggregate # noqa: E501
|
||||
else:
|
||||
triton = TritonPlaceholder()
|
||||
tl = TritonLanguagePlaceholder()
|
||||
tldevice = TritonLanguagePlaceholder()
|
||||
gluon = TritonLanguagePlaceholder()
|
||||
gl = TritonLanguagePlaceholder()
|
||||
aggregate = TritonLanguagePlaceholder()
|
||||
|
||||
from vllm.triton_utils.tensor_descriptor import use_tensor_descriptor
|
||||
|
||||
@@ -29,5 +35,8 @@ __all__ = [
|
||||
"tldevice",
|
||||
"LOG2E",
|
||||
"LOGE2",
|
||||
"gluon",
|
||||
"gl",
|
||||
"aggregate",
|
||||
"use_tensor_descriptor",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user