diff --git a/benchmarks/kernels/benchmark_inkling_qkvr_prep.py b/benchmarks/kernels/benchmark_inkling_qkvr_prep.py new file mode 100644 index 0000000000..8aa1233c50 --- /dev/null +++ b/benchmarks/kernels/benchmark_inkling_qkvr_prep.py @@ -0,0 +1,176 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import statistics + +import torch +from tabulate import tabulate + +from vllm.models.inkling.nvidia.ops import qkvr_prep +from vllm.utils.argparse_utils import FlexibleArgumentParser + + +def make_inputs(tokens: int, tp_size: int, is_local: bool): + torch.manual_seed(0) + num_q_heads = 64 // tp_size + num_kv_heads = (16 if is_local else 8) // tp_size + head_dim = 128 + d_rel = 16 + rel_extent = 512 if is_local else 1024 + page_size = 16 + num_blocks = (tokens + page_size - 1) // page_size + q_width = num_q_heads * head_dim + kv_width = num_kv_heads * head_dim + r_width = num_q_heads * d_rel + device = "cuda" + + qkvr = torch.randn( + tokens, + q_width + 2 * kv_width + r_width, + device=device, + dtype=torch.bfloat16, + ) + k_weight = torch.randn(kv_width, 4, device=device, dtype=torch.bfloat16) + v_weight = torch.randn_like(k_weight) + q_norm_weight = torch.randn(head_dim, device=device, dtype=torch.bfloat16) + k_norm_weight = torch.randn_like(q_norm_weight) + rel_proj = torch.randn(d_rel, rel_extent, device=device, dtype=torch.bfloat16) + conv_cache = torch.zeros( + num_blocks, + num_kv_heads, + page_size, + 2 * head_dim, + device=device, + dtype=torch.bfloat16, + ) + key_cache = torch.empty( + num_blocks, + page_size, + num_kv_heads, + head_dim, + device=device, + dtype=torch.bfloat16, + ) + value_cache = torch.empty_like(key_cache) + positions = torch.arange(tokens, device=device, dtype=torch.int64) + block_table = torch.arange(num_blocks, device=device, dtype=torch.int32)[None] + seq_idx = torch.zeros(tokens, device=device, dtype=torch.int32) + slots = torch.arange(tokens, device=device, dtype=torch.int64) + query_start = torch.zeros(tokens, device=device, dtype=torch.int32) + log_scaling = None + if not is_local: + effective_n = (positions + 1).to(torch.float32) + log_scaling = 1.0 + 0.1 * torch.log(torch.clamp(effective_n / 128000, min=1.0)) + return ( + qkvr, + k_weight, + v_weight, + q_norm_weight, + k_norm_weight, + rel_proj, + 1e-6, + num_q_heads, + num_kv_heads, + head_dim, + d_rel, + conv_cache, + key_cache, + value_cache, + positions, + block_table, + seq_idx, + slots, + query_start, + slots, + 0, + head_dim, + page_size, + log_scaling, + ) + + +def capture(implementation, inputs): + outputs = [] + + def run(): + outputs[:] = implementation.fused_qkvr_prep(*inputs) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + run() + torch.cuda.current_stream().wait_stream(stream) + torch.accelerator.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + torch.accelerator.synchronize() + return graph, outputs + + +def time_graph(graph: torch.cuda.CUDAGraph, warmup: int, repeats: int) -> float: + for _ in range(warmup): + graph.replay() + torch.accelerator.synchronize() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(repeats): + graph.replay() + end.record() + end.synchronize() + return start.elapsed_time(end) * 1000 / repeats + + +def benchmark(inputs, args) -> float: + graph, _ = capture(qkvr_prep, inputs) + return statistics.median( + time_graph(graph, args.warmup, args.repeats) for _ in range(args.trials) + ) + + +@torch.inference_mode() +def main(args): + rows = [] + for tp_size in args.tp_sizes: + for tokens in args.tokens: + for is_local in (True, False): + triton_us = benchmark(make_inputs(tokens, tp_size, is_local), args) + rows.append( + [ + tp_size, + tokens, + "local" if is_local else "global", + triton_us, + ] + ) + + print("Inkling QKVR prep (CUDA graph, median latency)") + print( + tabulate( + rows, + headers=[ + "TP", + "tokens", + "scope", + "Triton (us)", + ], + floatfmt=("d", "d", "", ".2f"), + ) + ) + + +if __name__ == "__main__": + parser = FlexibleArgumentParser() + parser.add_argument( + "--tokens", + type=int, + nargs="+", + default=[1 << power for power in range(15)], + ) + parser.add_argument("--tp-sizes", type=int, nargs="+", default=[4, 8]) + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--repeats", type=int, default=200) + parser.add_argument("--trials", type=int, default=5) + main(parser.parse_args()) diff --git a/tests/models/inkling/rocm/conftest.py b/tests/models/inkling/rocm/conftest.py new file mode 100644 index 0000000000..bbf1e7e12d --- /dev/null +++ b/tests/models/inkling/rocm/conftest.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from vllm.platforms import current_platform + + +@pytest.fixture(autouse=True) +def require_rocm_cdna4() -> None: + if not current_platform.is_rocm(): + pytest.skip("requires ROCm") + + from vllm.platforms.rocm import on_gfx950 + + if not on_gfx950(): + pytest.skip("requires CDNA4") diff --git a/tests/models/inkling/rocm/test_model_alignment.py b/tests/models/inkling/rocm/test_model_alignment.py new file mode 100644 index 0000000000..3bf3859de8 --- /dev/null +++ b/tests/models/inkling/rocm/test_model_alignment.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm guards for shared Inkling model-definition contracts.""" + +from pathlib import Path + +import torch + +import vllm.models.inkling as inkling +from vllm.lora.utils import get_supported_lora_modules +from vllm.model_executor.models.interfaces import supports_lora +from vllm.models.inkling.amd import model as amd_model +from vllm.models.inkling.amd.mtp import InklingMTP + + +def test_backend_neutral_definitions_match_nvidia_reference() -> None: + inkling_dir = Path(amd_model.__file__).parents[1] + for filename in ("logits_processor.py", "mtp.py"): + assert (inkling_dir / "amd" / filename).read_bytes() == ( + inkling_dir / "nvidia" / filename + ).read_bytes() + + +def test_rocm_exports_its_aligned_mtp_implementation() -> None: + assert inkling.InklingMTP is InklingMTP + + +def test_rocm_model_exposes_the_upstream_lora_contract() -> None: + model_cls = amd_model._TmlForCausalLMBase + + assert supports_lora(model_cls) + assert model_cls.packed_modules_mapping == { + "qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"], + "w13": ["w1", "w3"], + } + assert model_cls.embedding_modules == {"lm_head": "output_embeddings"} + + model = torch.nn.Module() + model.embedding_modules = model_cls.embedding_modules + supported = get_supported_lora_modules(model) + assert "embed_tokens" not in supported + assert "lm_head" in supported + + +def test_lightseek_bundled_adapter_weights_remain_opt_in() -> None: + mapper = amd_model._TmlForCausalLMBase.hf_to_vllm_mapper + weight = torch.empty(1) + name, mapped_weight = next( + iter( + mapper.apply([("language_model.layers.3.attn.wq_du.lora_A.weight", weight)]) + ) + ) + + assert name == "model.layers.3.attn.qkvr.lora_A.weight" + assert mapped_weight.shard_id == 0 + assert amd_model._is_peft_adapter_weight(name) + + lm_head_name = mapper.apply_list(["language_model.lm_head.lora_B.weight"]) + assert lm_head_name == ["lm_head.lora_B.weight"] diff --git a/tests/models/inkling/rocm/test_mxfp4_load.py b/tests/models/inkling/rocm/test_mxfp4_load.py new file mode 100644 index 0000000000..00ff2f4018 --- /dev/null +++ b/tests/models/inkling/rocm/test_mxfp4_load.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for Quark OCP MXFP4 weights loaded into AITER MoE layouts.""" + +from types import SimpleNamespace + +import torch +from torch import nn + +from vllm.models.inkling.amd.moe import InklingMoE + + +def _fake_moe() -> tuple[InklingMoE, SimpleNamespace]: + moe = InklingMoE.__new__(InklingMoE) + nn.Module.__init__(moe) + moe.n_routed_experts = 4 + routed = SimpleNamespace( + moe_config=SimpleNamespace( + moe_parallel_config=SimpleNamespace(tp_rank=1, tp_size=2) + ), + # Logical intermediate per rank is 3; AITER pads it to 4. + w13_weight=nn.Parameter(torch.zeros(4, 8, 2, dtype=torch.uint8), False), + w2_weight=nn.Parameter(torch.zeros(4, 5, 3, dtype=torch.uint8), False), + w13_weight_scale=nn.Parameter(torch.ones(4, 8, 2, dtype=torch.uint8), False), + w2_weight_scale=nn.Parameter(torch.ones(4, 5, 3, dtype=torch.uint8), False), + ) + moe.experts = SimpleNamespace(routed_experts=routed) + moe._local_expert_slots = ( # type: ignore[method-assign] + lambda: dict(enumerate(range(4))) + ) + return moe, routed + + +def test_loads_logical_tp_shards_and_preserves_aiter_padding(): + moe, routed = _fake_moe() + + w13 = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4, 12, 2) + w2 = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4, 5, 4) + moe.load_expert_weight("experts.w13_weight", w13) + moe.load_expert_weight("experts.w2_weight", w2) + + # TP rank 1 consumes the second six interleaved gate/up rows. AITER keeps + # one padding row in each destination half and one padding w2 column. + torch.testing.assert_close(routed.w13_weight[:, :3], w13[:, 6:12:2]) + torch.testing.assert_close(routed.w13_weight[:, 4:7], w13[:, 7:12:2]) + assert torch.count_nonzero(routed.w13_weight[:, 3]) == 0 + assert torch.count_nonzero(routed.w13_weight[:, 7]) == 0 + torch.testing.assert_close(routed.w2_weight[:, :, :2], w2[:, :, 2:4]) + assert torch.count_nonzero(routed.w2_weight[:, :, 2]) == 0 + + +def test_unflattens_quark_scales_and_preserves_scale_padding(): + moe, routed = _fake_moe() + + flat_w13_scale = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4 * 12, 2) + flat_w2_scale = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4 * 5, 4) + moe.load_expert_weight("experts.w13_weight_scale", flat_w13_scale) + moe.load_expert_weight("experts.w2_weight_scale", flat_w2_scale) + + w13_scale = flat_w13_scale.view(4, 12, 2) + w2_scale = flat_w2_scale.view(4, 5, 4) + torch.testing.assert_close(routed.w13_weight_scale[:, :3], w13_scale[:, 6:12:2]) + torch.testing.assert_close(routed.w13_weight_scale[:, 4:7], w13_scale[:, 7:12:2]) + assert torch.all(routed.w13_weight_scale[:, 3] == 1) + assert torch.all(routed.w13_weight_scale[:, 7] == 1) + torch.testing.assert_close(routed.w2_weight_scale[:, :, :2], w2_scale[:, :, 2:4]) + assert torch.all(routed.w2_weight_scale[:, :, 2] == 1) diff --git a/tests/models/inkling/rocm/test_rel_attention.py b/tests/models/inkling/rocm/test_rel_attention.py new file mode 100644 index 0000000000..c154faca34 --- /dev/null +++ b/tests/models/inkling/rocm/test_rel_attention.py @@ -0,0 +1,425 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Correctness tests for Inkling's ROCm relative-bias paged attention.""" + +import pytest +import torch + +from vllm.models.inkling.amd.ops.fa4_rel_attention import ( + bucket_max_seqlen_q, + inkling_fa4_rel_attention, + use_gfx950_gluon_decode, + use_gfx950_gluon_extend, +) +from vllm.models.inkling.amd.ops.rel_attention_decode import ( + decode_split_count, + use_split_kv_decode, +) + +HEAD_DIM = 128 +DTYPE = torch.bfloat16 + + +def _reference( + q: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + rel_logits: torch.Tensor, + block_table: torch.Tensor, + q_lens: list[int], + kv_lens: list[int], + rel_extent: int, + window_left: int | None, +) -> torch.Tensor: + num_heads = q.shape[1] + num_kv_heads = key_cache.shape[2] + gqa_group = num_heads // num_kv_heads + out: list[torch.Tensor] = [] + q_start = 0 + + for req, (q_len, kv_len) in enumerate(zip(q_lens, kv_lens)): + page_size = key_cache.shape[1] + num_pages = (kv_len + page_size - 1) // page_size + page_ids = block_table[req, :num_pages].long() + k = key_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len] + v = value_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len] + k = k.float().repeat_interleave(gqa_group, dim=1) + v = v.float().repeat_interleave(gqa_group, dim=1) + + q_req = q[q_start : q_start + q_len].float() + rel_req = rel_logits[q_start : q_start + q_len].float() + scores = torch.einsum("qhd,khd->hqk", q_req, k) / HEAD_DIM + + q_pos = torch.arange(q_len, device=q.device)[:, None] + kv_len - q_len + k_pos = torch.arange(kv_len, device=q.device)[None, :] + distance = q_pos - k_pos + rel_idx = distance.clamp(0, rel_extent - 1) + bias = rel_req.permute(1, 0, 2).gather( + 2, rel_idx[None].expand(num_heads, -1, -1) + ) + in_rel_extent = (distance >= 0) & (distance < rel_extent) + scores += torch.where(in_rel_extent[None], bias, 0.0) + + masked = distance < 0 + if window_left is not None: + masked |= distance > window_left + scores.masked_fill_(masked[None], float("-inf")) + out.append(torch.einsum("hqk,khd->qhd", scores.softmax(-1), v)) + q_start += q_len + + return torch.cat(out).to(DTYPE) + + +def test_query_length_bucket(): + assert bucket_max_seqlen_q(1) == 1 + assert bucket_max_seqlen_q(17) == 32 + assert bucket_max_seqlen_q(33) == 64 + + +def test_split_kv_dispatch_thresholds(): + assert not use_split_kv_decode( + max_query_len=4, + max_kv_len=65536, + page_size=128, + window_left=-1, + ) + assert not use_split_kv_decode( + max_query_len=1, + max_kv_len=4096, + page_size=16, + window_left=-1, + ) + assert use_split_kv_decode( + max_query_len=1, + max_kv_len=8192, + page_size=16, + window_left=-1, + ) + assert use_split_kv_decode( + max_query_len=1, + max_kv_len=512, + page_size=128, + window_left=511, + ) + assert decode_split_count(512, 511) == 4 + assert decode_split_count(65536, -1) == 32 + + +def test_gfx950_gluon_dispatch_is_strict(monkeypatch: pytest.MonkeyPatch): + import vllm.models.inkling.amd.ops.fa4_rel_attention as rel_attention + + monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True) + assert use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128) + assert not use_gfx950_gluon_decode(max_query_len=1, page_size=16, head_dim=128) + assert not use_gfx950_gluon_decode(max_query_len=4, page_size=128, head_dim=128) + assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=256) + + monkeypatch.setattr(rel_attention, "on_gfx950", lambda: False) + assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128) + + monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True) + assert use_gfx950_gluon_extend( + max_query_len=4, + max_kv_len=8192, + page_size=128, + head_dim=128, + window_left=-1, + ) + assert not use_gfx950_gluon_extend( + max_query_len=4, + max_kv_len=8192, + page_size=128, + head_dim=128, + window_left=511, + ) + assert not use_gfx950_gluon_extend( + max_query_len=4, + max_kv_len=8192, + page_size=16, + head_dim=128, + window_left=-1, + ) + + +@pytest.mark.parametrize( + ("page_size", "window_left", "max_kv_len"), + [ + (16, None, None), + (16, 15, None), + (128, None, 8192), + ], +) +@torch.inference_mode() +def test_ragged_multi_page_relative_attention( + page_size: int, + window_left: int | None, + max_kv_len: int | None, +): + """Covers full/chunked prefill, decode, GQA, and the local window.""" + torch.manual_seed(19 + int(window_left is not None)) + device = "cuda" + q_lens = [17, 1] + kv_lens = [35, 80] + num_heads = 8 + num_kv_heads = 2 + rel_extent = 16 + max_pages = max((length + page_size - 1) // page_size for length in kv_lens) + + q = torch.randn(sum(q_lens), num_heads, HEAD_DIM, device=device) + q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE) + key_cache = torch.randn( + 1 + len(q_lens) * max_pages, + page_size, + num_kv_heads, + HEAD_DIM, + device=device, + ) + key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE) + value_cache = torch.randn_like(key_cache) + rel_logits = torch.randn( + sum(q_lens), num_heads, rel_extent, device=device, dtype=DTYPE + ) + + block_table = torch.stack( + [ + torch.arange(1 + req * max_pages, 1 + (req + 1) * max_pages) + for req in range(len(q_lens)) + ] + ).to(device=device, dtype=torch.int32) + cu_seqlens_q = torch.tensor( + [0, *torch.tensor(q_lens).cumsum(0).tolist()], + device=device, + dtype=torch.int32, + ) + cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32) + window_size = (-1, -1) if window_left is None else (window_left, 0) + + preallocated = torch.empty_like(q) + actual = inkling_fa4_rel_attention( + q, + key_cache, + value_cache, + block_table=block_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=cu_seqlens_q, + max_seqlen_q=bucket_max_seqlen_q(max(q_lens)), + softmax_scale=1.0 / HEAD_DIM, + causal=True, + window_size=window_size, + rel_extent=rel_extent, + rel_logits=rel_logits, + max_kv_len=max_kv_len, + out=preallocated, + ) + assert actual.data_ptr() == preallocated.data_ptr() + + expected = _reference( + q, + key_cache, + value_cache, + rel_logits, + block_table, + q_lens, + kv_lens, + rel_extent, + window_left, + ) + torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2) + + +@pytest.mark.parametrize( + ("page_size", "kv_lens", "num_kv_heads", "rel_extent", "window_left"), + [ + (16, [257, 513], 1, 64, None), + (128, [257, 512], 2, 512, 511), + ], +) +@torch.inference_mode() +def test_split_kv_decode_matches_reference( + page_size: int, + kv_lens: list[int], + num_kv_heads: int, + rel_extent: int, + window_left: int | None, +): + """Covers page-16 split-KV and page-128 gfx950 Gluon decode.""" + torch.manual_seed(41 + page_size) + device = "cuda" + batch_size = len(kv_lens) + q_lens = [1] * batch_size + num_heads = 8 + max_pages = max((length + page_size - 1) // page_size for length in kv_lens) + + q = torch.randn(batch_size, num_heads, HEAD_DIM, device=device) + q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE) + key_cache = torch.randn( + batch_size * max_pages, + page_size, + num_kv_heads, + HEAD_DIM, + device=device, + ) + key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE) + value_cache = torch.randn_like(key_cache) + rel_logits = torch.randn( + batch_size, + num_heads, + rel_extent, + device=device, + dtype=DTYPE, + ) + block_table = torch.arange( + batch_size * max_pages, + device=device, + dtype=torch.int32, + ).view(batch_size, max_pages) + cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32) + cu_seqlens_q = torch.arange( + batch_size + 1, + device=device, + dtype=torch.int32, + ) + window_size = (-1, -1) if window_left is None else (window_left, 0) + max_kv_len = 8192 if page_size == 16 else max(kv_lens) + + actual = inkling_fa4_rel_attention( + q, + key_cache, + value_cache, + block_table=block_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=cu_seqlens_q, + max_seqlen_q=1, + softmax_scale=1.0 / HEAD_DIM, + causal=True, + window_size=window_size, + rel_extent=rel_extent, + rel_logits=rel_logits, + max_kv_len=max_kv_len, + ) + expected = _reference( + q, + key_cache, + value_cache, + rel_logits, + block_table, + q_lens, + kv_lens, + rel_extent, + window_left, + ) + torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2) + + +@pytest.mark.parametrize( + ( + "q_len", + "kv_len", + "num_kv_heads", + "rel_extent", + "window_left", + "gluon_enabled", + ), + [ + (1, 513, 2, 512, 511, True), + (1, 513, 2, 512, 511, False), + (4, 8192, 1, 1024, None, True), + ], +) +@torch.inference_mode() +def test_gfx950_page128_packed_kv_views_match_reference( + monkeypatch: pytest.MonkeyPatch, + q_len: int, + kv_len: int, + num_kv_heads: int, + rel_extent: int, + window_left: int | None, + gluon_enabled: bool, +): + """Cover both Gluon and split-KV against vLLM's packed KV allocation.""" + monkeypatch.setenv( + "INKLING_GFX950_GLUON", + "1" if gluon_enabled else "0", + ) + torch.manual_seed(71 + q_len) + device = "cuda" + page_size = 128 + batch_size = 2 + num_heads = 8 + q_lens = [q_len] * batch_size + kv_lens = [kv_len] * batch_size + pages_per_req = (kv_len + page_size - 1) // page_size + + q = torch.randn( + batch_size * q_len, + num_heads, + HEAD_DIM, + device=device, + dtype=DTYPE, + ) + q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE) + + # FlashAttentionBackend allocates logical [block, head, page, 2 * dim]. + # Inkling transposes to [block, page, head, 2 * dim] and splits K/V, + # producing non-contiguous views whose page/head strides include both. + packed_kv = torch.randn( + batch_size * pages_per_req, + num_kv_heads, + page_size, + 2 * HEAD_DIM, + device=device, + dtype=DTYPE, + ) + key_cache, value_cache = packed_kv.transpose(1, 2).split(HEAD_DIM, dim=-1) + assert not key_cache.is_contiguous() + assert key_cache.stride() == value_cache.stride() + + rel_logits = torch.randn( + batch_size * q_len, + num_heads, + rel_extent, + device=device, + dtype=DTYPE, + ) + block_table = torch.arange( + batch_size * pages_per_req, + device=device, + dtype=torch.int32, + ).view(batch_size, pages_per_req) + cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32) + cu_seqlens_q = torch.arange( + 0, + batch_size * q_len + 1, + q_len, + device=device, + dtype=torch.int32, + ) + window_size = (-1, -1) if window_left is None else (window_left, 0) + + actual = inkling_fa4_rel_attention( + q, + key_cache, + value_cache, + block_table=block_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=cu_seqlens_q, + max_seqlen_q=q_len, + softmax_scale=1.0 / HEAD_DIM, + causal=True, + window_size=window_size, + rel_extent=rel_extent, + rel_logits=rel_logits, + max_kv_len=kv_len, + ) + expected = _reference( + q, + key_cache, + value_cache, + rel_logits, + block_table, + q_lens, + kv_lens, + rel_extent, + window_left, + ) + torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2) diff --git a/tests/models/inkling/rocm/test_rocm_mtp_input_fusion.py b/tests/models/inkling/rocm/test_rocm_mtp_input_fusion.py new file mode 100644 index 0000000000..948c3a7f2a --- /dev/null +++ b/tests/models/inkling/rocm/test_rocm_mtp_input_fusion.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm numerical-equivalence tests for the fused MTP depth-layer input kernel. + +``embed_dual_rmsnorm_cat`` must match the unfused module sequence numerically: +each rmsnorm computes in fp32 and rounds to bf16 at the same points as the +vendored ``rmsnorm`` kernel (including the bf16 round-trip between the +chained backbone embed_norm and the depth embed_norm), and the fused row +gather matches ``F.embedding``. +""" + +from typing import cast + +import pytest +import torch + +from vllm.models.inkling.amd.ops.norm import ( + embed_dual_rmsnorm_cat, + embed_rmsnorm, + rmsnorm, +) + +EPS = 1e-6 +VOCAB = 4096 +BF16_ATOL = torch.finfo(torch.bfloat16).eps + + +def _assert_bf16_close(actual: torch.Tensor, expected: torch.Tensor) -> None: + # Different Triton reduction schedules can move a small number of outputs + # by one bf16 epsilon while remaining numerically equivalent. + torch.testing.assert_close(actual, expected, rtol=0, atol=BF16_ATOL) + + +def _ref(hidden, w_h, w_e, emb, w_pre=None): + if w_pre is not None: + emb = rmsnorm(emb, w_pre, EPS) + return torch.cat([rmsnorm(hidden, w_h, EPS), rmsnorm(emb, w_e, EPS)], dim=-1) + + +@pytest.mark.parametrize("n", [1536, 6144]) +@pytest.mark.parametrize("t", [0, 1, 7, 256]) +@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64]) +def test_embed_dual_rmsnorm_cat(n: int, t: int, ids_dtype: torch.dtype) -> None: + torch.manual_seed(0) + dev = "cuda" + table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16) + w_h = torch.randn(n, device=dev).to(torch.bfloat16) + w_e = (1 + 0.01 * torch.randn(n, device=dev)).to(torch.bfloat16) + w_pre = torch.randn(n, device=dev).to(torch.bfloat16) + ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype) + hidden = (torch.randn(t, n, device=dev) * 2).to(torch.bfloat16) + emb = table[ids.long()] + + # Fused gather + chained backbone pre-norm (the decode draft-step path). + out = embed_dual_rmsnorm_cat( + hidden, + w_h, + w_e, + EPS, + input_ids=ids, + embed_table=table, + pre_norm_weight=w_pre, + ) + assert out.shape == (t, 2 * n) + _assert_bf16_close(out, _ref(hidden, w_h, w_e, emb, w_pre)) + + # Precomputed embeds, no pre-norm (draft prefill with target-merged MM + # embeddings, already backbone-normed). + out = embed_dual_rmsnorm_cat(hidden, w_h, w_e, EPS, embeds=emb) + _assert_bf16_close(out, _ref(hidden, w_h, w_e, emb)) + + # Fused gather, no pre-norm (use_embed_norm=False). + out = embed_dual_rmsnorm_cat( + hidden, w_h, w_e, EPS, input_ids=ids, embed_table=table + ) + _assert_bf16_close(out, _ref(hidden, w_h, w_e, emb)) + + +@pytest.mark.parametrize("n", [1536, 6144]) +@pytest.mark.parametrize("t", [0, 1, 7, 256]) +@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64]) +def test_embed_rmsnorm(n: int, t: int, ids_dtype: torch.dtype) -> None: + torch.manual_seed(0) + dev = "cuda" + table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16) + w = torch.randn(n, device=dev).to(torch.bfloat16) + ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype) + ref_emb = table[ids.long()] + + # Gather + embed_norm (base model / MTP prefill embed path). + out = cast(torch.Tensor, embed_rmsnorm(ids, table, w, EPS)) + assert out.shape == (t, n) + _assert_bf16_close(out, rmsnorm(ref_emb, w, EPS) if t else ref_emb) + + # Pure gather (use_embed_norm=False / replicated module forward). + out = cast(torch.Tensor, embed_rmsnorm(ids, table, None, EPS)) + assert torch.equal(out, ref_emb) + + # Chained first-layer attn_norm (the target text-path forward): one launch + # emits both the residual and layer 0's normed attention input. + w_chain = (1 + 0.05 * torch.randn(n, device=dev)).to(torch.bfloat16) + res, attn_in = cast( + tuple[torch.Tensor, torch.Tensor], + embed_rmsnorm(ids, table, w, EPS, chain_weight=w_chain), + ) + ref_res = rmsnorm(ref_emb, w, EPS) if t else ref_emb + _assert_bf16_close(res, ref_res) + _assert_bf16_close(attn_in, rmsnorm(ref_res, w_chain, EPS) if t else ref_res) + + # Chained without embed_norm (use_embed_norm=False). + res, attn_in = cast( + tuple[torch.Tensor, torch.Tensor], + embed_rmsnorm(ids, table, None, EPS, chain_weight=w_chain), + ) + assert torch.equal(res, ref_emb) + _assert_bf16_close(attn_in, rmsnorm(ref_emb, w_chain, EPS) if t else ref_emb) diff --git a/tests/models/inkling/rocm/test_sconv_cache_layout.py b/tests/models/inkling/rocm/test_sconv_cache_layout.py new file mode 100644 index 0000000000..eeefb877d8 --- /dev/null +++ b/tests/models/inkling/rocm/test_sconv_cache_layout.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from torch import nn + +from vllm.models.inkling.amd.sconv_swa_attn import InklingConvState + + +def test_runtime_sconv_block_size_tracks_unified_cache_page(): + """The cache planner may enlarge W=4 blocks to match attention pages.""" + owner = InklingConvState.__new__(InklingConvState) + nn.Module.__init__(owner) + owner.block_size = 4 + + owner.kv_cache = torch.tensor([]) + assert owner.cache_block_size == 4 + + owner.kv_cache = torch.empty(2, 1, 32, 1024) + assert owner.cache_block_size == 32 diff --git a/vllm/models/inkling/__init__.py b/vllm/models/inkling/__init__.py index 32e58905e2..4726d68e48 100644 --- a/vllm/models/inkling/__init__.py +++ b/vllm/models/inkling/__init__.py @@ -2,12 +2,21 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import TYPE_CHECKING +from vllm.platforms import current_platform + if TYPE_CHECKING: - from .nvidia.model import ( - InklingForCausalLM, - InklingForConditionalGeneration, - ) - from .nvidia.mtp import InklingMTP + if current_platform.is_rocm(): + from .amd.model import ( + InklingForCausalLM, + InklingForConditionalGeneration, + ) + from .amd.mtp import InklingMTP as InklingMTP + else: + from .nvidia.model import ( + InklingForCausalLM, + InklingForConditionalGeneration, + ) + from .nvidia.mtp import InklingMTP as InklingMTP __all__ = [ "InklingForConditionalGeneration", @@ -18,11 +27,23 @@ __all__ = [ def __getattr__(name: str): if name == "InklingMTP": - from .nvidia import mtp + if current_platform.is_rocm(): + from .amd import mtp as amd_mtp + + return amd_mtp.InklingMTP + + from .nvidia import mtp as nvidia_mtp + + return nvidia_mtp.InklingMTP - return mtp.InklingMTP if name in __all__: - from .nvidia import model + if current_platform.is_rocm(): + from .amd import model as amd_model + + return getattr(amd_model, name) + + from .nvidia import model as nvidia_model + + return getattr(nvidia_model, name) - return getattr(model, name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/vllm/models/inkling/amd/__init__.py b/vllm/models/inkling/amd/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/inkling/amd/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/inkling/amd/attention.py b/vllm/models/inkling/amd/attention.py new file mode 100644 index 0000000000..edbc64bd81 --- /dev/null +++ b/vllm/models/inkling/amd/attention.py @@ -0,0 +1,328 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +from typing import cast + +import torch +from torch import nn + +from vllm.compilation.breakable_cudagraph import eager_break_during_capture +from vllm.config import VllmConfig, get_current_vllm_config +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.utils.torch_utils import ( + canonicalize_singleton_dim_strides, + kv_cache_dtype_str_to_dtype, +) +from vllm.v1.attention.backend import AttentionBackend +from vllm.v1.attention.backends.flash_attn import ( + FlashAttentionBackend, + FlashAttentionMetadata, +) +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheSpec, + SlidingWindowSpec, +) + +from ..configs import InklingModelConfig +from .layernorm import InklingRMSNorm +from .ops.fa4_rel_attention import ( + bucket_max_seqlen_q, + inkling_fa4_num_splits, + inkling_fa4_rel_attention, +) +from .ops.fa4_warmup import InklingFA4WarmupConfig, register_fa4_warmup +from .ops.qkvr_prep import fused_qkvr_prep +from .sconv_swa_attn import _K, _V, InklingConvState, InklingSconvMetadata +from .short_conv import InklingShortConv + + +def compute_log_scaling_tau( + positions: torch.Tensor, n_floor: int, alpha: float +) -> torch.Tensor: + effective_n = (positions + 1).to(torch.float32) + return 1.0 + alpha * torch.log(torch.clamp(effective_n / float(n_floor), min=1.0)) + + +class RelLogitsProj(nn.Module): + """Project the per-head relative branch ``r`` to per-distance logits.""" + + def __init__(self, d_rel: int, rel_extent: int) -> None: + super().__init__() + self.d_rel = d_rel + self.rel_extent = rel_extent + self.proj = nn.Parameter(torch.empty(d_rel, rel_extent), requires_grad=False) + + def forward(self, r_out: torch.Tensor) -> torch.Tensor: + # r_out: (T, num_heads, d_rel) -> (T, num_heads, rel_extent) + return torch.einsum("thd,de->the", r_out, self.proj) + + +class InklingAttention(nn.Module, AttentionLayerBase): + def __init__( + self, + config: InklingModelConfig, + *, + num_heads: int, + num_kv_heads: int, + head_dim: int, + rel_extent: int, + local_extent: int, + is_local: bool, + prefix: str, + quant_config: QuantizationConfig | None = None, + conv_owner: InklingConvState, + ) -> None: + super().__init__() + self.prefix = prefix + self.is_local = is_local + self.hidden_size = config.hidden_size + self.head_dim = head_dim + self.d_rel = config.d_rel + self.log_scaling_n_floor = config.log_scaling_n_floor + self.log_scaling_alpha = config.log_scaling_alpha + # q/k are per-head RMS-normed (unit norm), so Inkling scales by 1/head_dim. + self.scaling = 1.0 / head_dim + + tp_size = get_tensor_model_parallel_world_size() + self.num_total_heads = num_heads + self.num_total_kv_heads = num_kv_heads + assert self.num_total_heads % tp_size == 0 + self.num_heads = self.num_total_heads // tp_size + if self.num_total_kv_heads >= tp_size: + assert self.num_total_kv_heads % tp_size == 0 + else: + assert tp_size % self.num_total_kv_heads == 0 + self.num_kv_heads = max(1, self.num_total_kv_heads // tp_size) + # When tp_size > num_kv_heads the K/V projections are padded up to + # tp_size heads so each rank gets at least one (GQA replication). + kv_total_for_sizing = max(self.num_total_kv_heads, tp_size) + + self.qkvr = MergedColumnParallelLinear( + input_size=config.hidden_size, + output_sizes=[ + head_dim * self.num_total_heads, + head_dim * kv_total_for_sizing, + head_dim * kv_total_for_sizing, + self.d_rel * self.num_total_heads, + ], + bias=config.q_bias, + quant_config=quant_config, + prefix=f"{prefix}.qkvr", + ) + self.wo_ud = RowParallelLinear( + input_size=head_dim * self.num_total_heads, + output_size=config.hidden_size, + bias=config.o_bias, + quant_config=quant_config, + # reduce_results=False: the partial output is all-reduced below + # (one-shot custom AR) so the attention-output sconv can run on the + # full hidden width fused with the residual add + rmsnorm. + reduce_results=False, + prefix=f"{prefix}.wo_ud", + ) + self.rel_extent = local_extent if is_local else rel_extent + self.local_extent = local_extent if is_local else None + self.rel_logits_proj = RelLogitsProj(self.d_rel, self.rel_extent) + self.q_norm = InklingRMSNorm(head_dim, eps=config.rms_norm_eps) + self.k_norm = InklingRMSNorm(head_dim, eps=config.rms_norm_eps) + + # Short convolution on the K/V streams (per-head-width, TP sharded), + # applied after the qkvr projection and before q/k norm. + kv_conv_dim = self.num_kv_heads * head_dim + self.conv_owner = conv_owner + self.k_sconv = InklingShortConv( + kv_conv_dim, config.sconv_kernel_size, owner=conv_owner, stream_idx=_K + ) + self.v_sconv = InklingShortConv( + kv_conv_dim, config.sconv_kernel_size, owner=conv_owner, stream_idx=_V + ) + + # FA4 left/right window; right=0 keeps it causal. local_extent-1 mirrors + # the source (sliding_window_size - 1). + self.window_size: tuple[int, int] = ( + (local_extent - 1, 0) if is_local else (-1, -1) + ) + # Static per-layer-type KV length bound for the split heuristic: local + # layers never see more than the sliding window. + vllm_config = get_current_vllm_config() + self._max_kv_len = ( + local_extent if is_local else vllm_config.model_config.max_model_len + ) + + # ---- KV-cache wiring (reuse FlashAttentionBackend for metadata) ---- + cache_config = vllm_config.cache_config + self.kv_cache_dtype = ( + cache_config.cache_dtype if cache_config is not None else "auto" + ) + self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype( + self.kv_cache_dtype, vllm_config.model_config + ) + self.register_buffer("k_scale", torch.ones((), dtype=torch.float32)) + self.register_buffer("v_scale", torch.ones((), dtype=torch.float32)) + + compilation_config = vllm_config.compilation_config + if prefix in compilation_config.static_forward_context: + raise ValueError(f"Duplicate layer name: {prefix}") + compilation_config.static_forward_context[prefix] = self + self.kv_cache = torch.tensor([]) # replaced by bind_kv_cache + + register_fa4_warmup( + InklingFA4WarmupConfig( + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + head_dim=self.head_dim, + rel_extent=self.rel_extent, + window_size=self.window_size, + is_local=self.is_local, + max_kv_len=self._max_kv_len, + dtype=vllm_config.model_config.dtype, + kv_dtype=self.kv_cache_torch_dtype, + block_size=vllm_config.cache_config.block_size, + max_num_reqs=vllm_config.scheduler_config.max_num_seqs, + max_num_batched_tokens=( + vllm_config.scheduler_config.max_num_batched_tokens + ), + ) + ) + + def get_attn_backend(self) -> type[AttentionBackend]: + return FlashAttentionBackend + + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + block_size = vllm_config.cache_config.block_size + if self.is_local: + assert self.local_extent is not None + return SlidingWindowSpec( + block_size=block_size, + num_kv_heads=self.num_kv_heads, + head_size=self.head_dim, + dtype=self.kv_cache_torch_dtype, + sliding_window=self.local_extent, + ) + return FullAttentionSpec( + block_size=block_size, + num_kv_heads=self.num_kv_heads, + head_size=self.head_dim, + dtype=self.kv_cache_torch_dtype, + ) + + def _split_kv_cache(self) -> tuple[torch.Tensor, torch.Tensor]: + key_cache, value_cache = self.kv_cache.transpose(1, 2).split( + self.head_dim, dim=-1 + ) + return ( + canonicalize_singleton_dim_strides(key_cache), + canonicalize_singleton_dim_strides(value_cache), + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + log_scaling: torch.Tensor | None = None, + ) -> torch.Tensor: + num_tokens = hidden_states.shape[0] + qkvr, _ = self.qkvr(hidden_states) + + attn_metadata = get_forward_context().attn_metadata + attn_output = torch.empty( + (num_tokens, self.num_heads, self.head_dim), + dtype=qkvr.dtype, + device=qkvr.device, + ) + if not isinstance(attn_metadata, dict): + attn_output.zero_() + else: + conv_meta = attn_metadata[self.conv_owner.prefix] + md = attn_metadata[self.prefix] + assert isinstance(conv_meta, InklingSconvMetadata) + fa_md = cast(FlashAttentionMetadata, md) + assert self.kv_cache.numel() > 0 + assert self.conv_owner.kv_cache.numel() > 0 + # One launch: K/V sconv (conv-cache insert + conv + residual), + # Q/K per-head rmsnorm, and the attention KV-cache write. K/V are + # consumed via the KV cache; only normed q is materialized. + key_cache, value_cache = self._split_kv_cache() + off_k, _ = self.conv_owner.stream_ranges[_K] + off_v, _ = self.conv_owner.stream_ranges[_V] + q, rel_logits = fused_qkvr_prep( + qkvr, + self.k_sconv.weight.squeeze(1), + self.v_sconv.weight.squeeze(1), + self.q_norm.weight, + self.k_norm.weight, + self.rel_logits_proj.proj, + self.q_norm.variance_epsilon, + self.num_heads, + self.num_kv_heads, + self.head_dim, + self.d_rel, + self.conv_owner.kv_cache, + key_cache, + value_cache, + positions, + conv_meta.block_table, + conv_meta.seq_idx, + conv_meta.slot_mapping, + conv_meta.query_start, + fa_md.slot_mapping, + off_k, + off_v, + self.conv_owner.cache_block_size, + log_scaling if not self.is_local else None, + ) + q = q.view(num_tokens, self.num_heads, self.head_dim) + self._attention(q, rel_logits, attn_output) + + flat = attn_output.view(num_tokens, -1) + output, _ = self.wo_ud(flat) + return output + + @eager_break_during_capture + def _attention( + self, + q: torch.Tensor, + rel_logits: torch.Tensor, + output: torch.Tensor, + ) -> None: + attn_metadata = get_forward_context().attn_metadata + assert isinstance(attn_metadata, dict) + md = cast(FlashAttentionMetadata, attn_metadata[self.prefix]) + + nt = md.num_actual_tokens + key_cache, value_cache = self._split_kv_cache() + max_seqlen_q = bucket_max_seqlen_q(md.max_query_len) + num_splits = inkling_fa4_num_splits( + is_local=self.is_local, + batch_size=md.seq_lens.shape[0], + max_query_len=max_seqlen_q, + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + max_kv_len=md.max_seq_len, + ) + inkling_fa4_rel_attention( + q[:nt], + key_cache, + value_cache, + block_table=md.block_table, + cache_seqlens=md.seq_lens, + cu_seqlens_q=md.query_start_loc, + max_seqlen_q=max_seqlen_q, + softmax_scale=self.scaling, + causal=True, + window_size=self.window_size, + rel_extent=self.rel_extent, + rel_logits=rel_logits[:nt], + num_splits=num_splits, + max_kv_len=md.max_seq_len, + out=output[:nt], + ) diff --git a/vllm/models/inkling/amd/layernorm.py b/vllm/models/inkling/amd/layernorm.py new file mode 100644 index 0000000000..18f9951f65 --- /dev/null +++ b/vllm/models/inkling/amd/layernorm.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling RMSNorm (no bias, weight-scaled), backed by the vendored Triton kernel.""" + +from __future__ import annotations + +import torch +from torch import nn + +from .ops import rmsnorm + + +class InklingRMSNorm(nn.Module): + def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + self.hidden_size = hidden_size + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if x.numel() == 0: + return x + original_shape = x.shape + x_2d = x.contiguous().view(-1, self.hidden_size) + y = rmsnorm(x_2d, self.weight, self.variance_epsilon) + return y.view(original_shape) diff --git a/vllm/models/inkling/amd/logits_processor.py b/vllm/models/inkling/amd/logits_processor.py new file mode 100644 index 0000000000..2e142c5961 --- /dev/null +++ b/vllm/models/inkling/amd/logits_processor.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling logits processor (muP + LoRA aware). + +Inkling divides the final logits by a muP width multiplier +(``logits_mup_width_multiplier``). This applies it two ways, depending on +whether an lm_head LoRA is attached: + +* No LoRA: fold ``1/mup`` into the lm_head GEMM alpha (fp32 epilogue) -- no + separate elementwise kernel, no extra rounding, no weight mutation. +* LoRA attached: the LoRA manager wraps this layer in + ``LogitsProcessorWithLoRA``, whose ``forward`` calls + ``type(base_layer).forward(self=wrapper)`` -- so this ``forward`` runs with + ``self`` bound to the wrapper. We detect that via ``base_layer`` and take the + LoRA path: run the wrapper's ``_get_logits`` (base logits + the lm_head LoRA + delta), then divide the full logits by the multiplier so the delta is scaled + too. muP thus composes with the LoRA delta, with the dispatch as the only + model-side branching. +""" + +from __future__ import annotations + +import torch + +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding + + +class InklingLogitsProcessor(LogitsProcessor): + """``LogitsProcessor`` that applies Inkling's muP logits width multiplier. + + Args: + vocab_size: Padded vocabulary size. + org_vocab_size: Unpadded vocabulary size (defaults to ``vocab_size``). + scale: Base logits scale (kept ``1.0`` for the served checkpoint). + logits_as_input: Whether the input is already logits. + soft_cap: Optional logit soft cap (``None`` for the served checkpoint). + logits_mup_width_multiplier: muP width divisor for the final logits; + ``None`` or ``0`` disables it. + """ + + def __init__( + self, + vocab_size: int, + org_vocab_size: int | None = None, + scale: float = 1.0, + logits_as_input: bool = False, + soft_cap: float | None = None, + logits_mup_width_multiplier: float | None = None, + ) -> None: + super().__init__( + vocab_size=vocab_size, + org_vocab_size=org_vocab_size, + scale=scale, + logits_as_input=logits_as_input, + soft_cap=soft_cap, + ) + self.logits_mup_width_multiplier = logits_mup_width_multiplier + self._logits_zero: torch.Tensor | None = None + + def forward( + self, + lm_head: VocabParallelEmbedding, + hidden_states: torch.Tensor, + embedding_bias: torch.Tensor | None = None, + ) -> torch.Tensor | None: + # ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper, + # which calls this forward with ``self`` bound to the wrapper. The + # wrapper is not an ``InklingLogitsProcessor`` instance, so dispatch + # ``_lora_forward`` explicitly through the base_layer's class (it + # provides ``_get_logits``/``logits_as_input``; only ``_lora_forward`` + # lives on this class). + if hasattr(self, "base_layer"): + return type(self.base_layer)._lora_forward( + self, lm_head, hidden_states, embedding_bias + ) + return self._base_forward(lm_head, hidden_states, embedding_bias) + + def _lora_forward( + self, + lm_head: VocabParallelEmbedding, + hidden_states: torch.Tensor, + embedding_bias: torch.Tensor | None = None, + ) -> torch.Tensor | None: + # ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits`` + # returns the base logits plus the lm_head LoRA delta. Apply the muP + # divisor on the full logits so the LoRA delta is scaled too. + mup_multiplier = self.base_layer.logits_mup_width_multiplier + mup = 1.0 / mup_multiplier if mup_multiplier else None + if self.logits_as_input: + logits = hidden_states + else: + logits = self._get_logits(hidden_states, lm_head, embedding_bias) + # TODO: fuse this multiplication + if logits is not None and mup: + assert self.base_layer.soft_cap is None + assert self.base_layer.scale == 1.0 + logits = logits * mup + return logits + + def _base_forward( + self, + lm_head: VocabParallelEmbedding, + hidden_states: torch.Tensor, + embedding_bias: torch.Tensor | None = None, + ) -> torch.Tensor | None: + mup = self.logits_mup_width_multiplier + if not mup: + return super().forward(lm_head, hidden_states, embedding_bias) + # Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue): + # no separate elementwise kernel, no bf16 rounding of scaled logits, and + # no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no + # soft cap, unit logits scale. + assert self.soft_cap is None + assert self.scale == 1.0 + w = lm_head.weight + if self._logits_zero is None: + self._logits_zero = w.new_zeros(1) + logits = torch.addmm( + self._logits_zero, + hidden_states, + w.t(), + beta=0.0, + alpha=1.0 / mup, + ) + logits = self._gather_logits(logits) + if logits is not None: + logits = logits[..., : self.org_vocab_size] + return logits diff --git a/vllm/models/inkling/amd/mlp.py b/vllm/models/inkling/amd/mlp.py new file mode 100644 index 0000000000..51deb38e64 --- /dev/null +++ b/vllm/models/inkling/amd/mlp.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling dense SwiGLU MLP (also used as the MoE shared expert). + +The checkpoint stores the gate/up projection as a single fused, *interleaved* +weight (``[gate0, up0, gate1, up1, ...]``), so we use a plain +``ColumnParallelLinear`` whose contiguous row-sharding keeps each gate/up pair +together, and an interleaved SwiGLU activation. +""" + +from __future__ import annotations + +import torch +from torch import nn + +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.quantization import QuantizationConfig + + +class InklingDenseMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + *, + use_global_scale: bool = False, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = ColumnParallelLinear( + hidden_size, + 2 * intermediate_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=False, + prefix=f"{prefix}.down_proj", + ) + if use_global_scale: + self.global_scale = nn.Parameter(torch.empty(1), requires_grad=False) + else: + self.global_scale = None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + from .ops import silu_and_mul_triton + + gate_up, _ = self.gate_up_proj(x) + x = silu_and_mul_triton(gate_up) + x, _ = self.down_proj(x) + if self.global_scale is not None: + x.mul_(self.global_scale) + # TP-partial output: the layer's reduce-scatter fallback consumes it. + return x diff --git a/vllm/models/inkling/amd/model.py b/vllm/models/inkling/amd/model.py new file mode 100644 index 0000000000..56ffb1adcd --- /dev/null +++ b/vllm/models/inkling/amd/model.py @@ -0,0 +1,669 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling model implementation for AMD GPUs.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import regex as re +import torch +from torch import nn + +from vllm.config import VllmConfig +from vllm.distributed import ( + get_pp_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_gather, + tensor_model_parallel_reduce_scatter, +) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.models.interfaces import ( + MultiModalEmbeddings, + SupportsLoRA, + SupportsMultiModal, + SupportsPP, +) +from vllm.model_executor.models.utils import ( + AutoWeightsLoader, + WeightsMapper, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) +from vllm.models.inkling.common.mm_preprocess import ( + InklingDummyInputsBuilder, + InklingMultiModalProcessor, + InklingProcessingInfo, + inkling_audio_enabled, + inkling_vision_enabled, +) +from vllm.models.inkling.common.towers import InklingAudio, InklingVision +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.sequence import IntermediateTensors + +from ..configs import InklingMMConfig, InklingModelConfig +from .attention import InklingAttention, compute_log_scaling_tau +from .layernorm import InklingRMSNorm +from .logits_processor import InklingLogitsProcessor +from .mlp import InklingDenseMLP +from .moe import InklingMoE +from .ops.norm import add_rmsnorm, embed_rmsnorm +from .sconv_swa_attn import _ATTN, _MLP, InklingConvState +from .short_conv import InklingShortConv + + +def _layer_id(name: str) -> int | None: + m = re.search(r"\.layers\.(\d+)\.", name) + return int(m.group(1)) if m else None + + +def _sconv_add_norm( + delta: torch.Tensor, + hidden: torch.Tensor, + sconv: InklingShortConv, + norm: InklingRMSNorm | None, + positions: torch.Tensor, +) -> tuple[torch.Tensor | None, torch.Tensor]: + """``h = hidden + sconv(TP-sum(delta)); y = rmsnorm(h)``. + + ROCm uses the portable collective path. The Lamport P2P implementation in + the NVIDIA model relies on CUDA GDC and CUDA-specific symmetric-memory + publication semantics which cannot be linked into a gfx950 HSACO.""" + norm_w = norm.weight if norm is not None else None + eps = norm.variance_epsilon if norm is not None else 0.0 + + # RCCL RS -> shard sconv -> AG -> fused add(+rmsnorm). + shard = tensor_model_parallel_reduce_scatter(delta, dim=-1) + shard = sconv(shard.contiguous(), positions) + full = tensor_model_parallel_all_gather(shard, dim=-1) + if norm is None: + return None, hidden + full + return add_rmsnorm(hidden, full, norm_w, eps) + + +class InklingDecoderLayer(nn.Module): + def __init__( + self, + config: InklingModelConfig, + layer_id: int, + is_local: bool, + quant_config: QuantizationConfig | None, + prefix: str, + force_dense_mlp: bool = False, + ) -> None: + super().__init__() + # Per-layer owner of the conv state as a paged SWA cache. The 4 sconv + # streams (K/V/attn/mlp) are packed head-major into one block and share + # it. Built first so the attention layer can wire its K/V sconv to it. + self.conv_state = InklingConvState( + num_kv_heads=( + config.swa_num_key_value_heads + if is_local + else config.num_key_value_heads + ), + head_dim=config.swa_head_dim if is_local else config.head_dim, + hidden_size=config.hidden_size, + kernel_size=config.sconv_kernel_size, + prefix=f"{prefix}.conv_state", + ) + self.attn_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.attn = InklingAttention( + config, + num_heads=( + config.swa_num_attention_heads + if is_local + else config.num_attention_heads + ), + num_kv_heads=( + config.swa_num_key_value_heads + if is_local + else config.num_key_value_heads + ), + head_dim=config.swa_head_dim if is_local else config.head_dim, + rel_extent=config.rel_extent, + local_extent=config.sliding_window_size, + is_local=is_local, + prefix=f"{prefix}.attn", + quant_config=quant_config, + conv_owner=self.conv_state, + ) + self.mlp_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + if force_dense_mlp or layer_id < config.dense_mlp_idx: + self.mlp: nn.Module = InklingDenseMLP( + hidden_size=config.hidden_size, + intermediate_size=config.dense_intermediate_size, + use_global_scale=config.use_global_scale, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + else: + # InklingMoE decides per layer (from the checkpoint exclude list) + # whether the routed experts are NVFP4 or bf16; the shared sink + # experts are always bf16. + self.mlp = InklingMoE( + config, + prefix=f"{prefix}.mlp", + quant_config=quant_config, + ) + + # Short convolution on the attention-output and MLP-output residual + # streams, hidden-sharded: the sublayer outputs are reduce-scattered + # to [T, H/tp], the sconv runs on the shard, and an all-gather + # restores the full residual — all fused with the residual add + next + # rmsnorm via the Lamport P2P kernels for decode-sized batches. + tp_size = get_tensor_model_parallel_world_size() + sconv_dim = config.hidden_size // tp_size + self.attn_sconv = InklingShortConv( + sconv_dim, config.sconv_kernel_size, owner=self.conv_state, stream_idx=_ATTN + ) + self.mlp_sconv = InklingShortConv( + sconv_dim, config.sconv_kernel_size, owner=self.conv_state, stream_idx=_MLP + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + pending: tuple[torch.Tensor | None, InklingShortConv] | None = None, + defer_mlp_add: bool = False, + attn_in: torch.Tensor | None = None, + log_scaling: torch.Tensor | None = None, + ) -> ( + torch.Tensor | tuple[torch.Tensor, tuple[torch.Tensor | None, InklingShortConv]] + ): + # The previous sublayer's (pre-reduce, pre-sconv) delta is folded in + # fused with its RS/sconv/AG and this layer's pre-attention rmsnorm. + # A None delta means the partials sit in the NVLS symm buffer. + if pending is None: + if attn_in is None: + # First layer; on the text path attn_norm comes fused with + # the embedding gather (chain_weight in embed_rmsnorm). + attn_in = self.attn_norm(hidden_states) + else: + attn_in, hidden_states = _sconv_add_norm( + pending[0], hidden_states, pending[1], self.attn_norm, positions + ) + attn_output = self.attn(positions, attn_in, log_scaling) + mlp_in, hidden_states = _sconv_add_norm( + attn_output, hidden_states, self.attn_sconv, self.mlp_norm, positions + ) + mlp_output = self.mlp(mlp_in) + if defer_mlp_add: + # Caller folds mlp_output (pre-reduce, pre-sconv) into the next + # fused sconv+add+rmsnorm. + return hidden_states, (mlp_output, self.mlp_sconv) + # Standalone (MTP) tail: finish the sublayer without a norm. + return _sconv_add_norm( + mlp_output, hidden_states, self.mlp_sconv, None, positions + )[1] + + +class InklingReplicatedEmbedding(nn.Module): + """Full-vocab embedding table replicated on every TP rank. + + Trades the full table per rank (~2.3 GiB at V=201k / H=6144 bf16, vs a + 1/tp shard) for no masked lookup and no per-lookup TP all-reduce — one + all-reduce per MTP draft step plus one per verify pass — and keeps the + full table on-rank for the fused gather+norm kernels (``embed_rmsnorm``, + ``embed_dual_rmsnorm_cat``). Bit-exact vs vocab-parallel: the all-reduce + there only ever summed one real row against exact zeros. The LM head + stays vocab-sharded. + """ + + def __init__(self, num_embeddings: int, embedding_dim: int) -> None: + super().__init__() + self.weight = nn.Parameter( + torch.empty(num_embeddings, embedding_dim, dtype=torch.get_default_dtype()), + requires_grad=False, + ) + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + return embed_rmsnorm(input_ids, self.weight, None, 0.0) + + +class InklingModel(nn.Module): + def __init__( + self, + *, + config: InklingModelConfig, + quant_config: QuantizationConfig | None, + prefix: str, + ) -> None: + super().__init__() + self.config = config + self.embed_tokens = InklingReplicatedEmbedding( + config.padded_vocab_size, config.hidden_size + ) + self.embed_norm = ( + InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + if config.use_embed_norm + else None + ) + local_ids = set(config.local_layer_ids) + + def get_layer(prefix: str) -> InklingDecoderLayer: + idx = _layer_id(prefix + ".") or int(prefix.split(".")[-1]) + return InklingDecoderLayer( + config, idx, idx in local_ids, quant_config, prefix + ) + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers" + ) + self.norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states"], config.hidden_size + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + # Row gather + embed_norm in one launch. + norm = self.embed_norm + return embed_rmsnorm( + input_ids, + self.embed_tokens.weight, + norm.weight if norm is not None else None, + norm.variance_epsilon if norm is not None else 0.0, + ) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor | IntermediateTensors: + attn_in0: torch.Tensor | None = None + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + # embed_norm was already applied when producing inputs_embeds. + hidden_states = inputs_embeds + else: + # Gather + embed_norm + the first layer's attn_norm, one launch. + norm = self.embed_norm + hidden_states, attn_in0 = embed_rmsnorm( + input_ids, + self.embed_tokens.weight, + norm.weight if norm is not None else None, + self.config.rms_norm_eps, + chain_weight=self.layers[self.start_layer].attn_norm.weight, + ) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + hidden_states = hidden_states.view(-1, hidden_states.shape[-1]) + log_scaling = None + if self.config.log_scaling_n_floor is not None: + log_scaling = compute_log_scaling_tau( + positions, + self.config.log_scaling_n_floor, + self.config.log_scaling_alpha, + ) + + pending: tuple[torch.Tensor | None, InklingShortConv] | None = None + for layer in self.layers[self.start_layer : self.end_layer]: + hidden_states, pending = layer( + positions, + hidden_states, + pending=pending, + defer_mlp_add=True, + attn_in=attn_in0, + log_scaling=log_scaling, + ) + attn_in0 = None + + if not get_pp_group().is_last_rank: + if pending is not None: + hidden_states = _sconv_add_norm( + pending[0], hidden_states, pending[1], None, positions + )[1] + return IntermediateTensors({"hidden_states": hidden_states}) + if pending is not None: + # Final RS/sconv/AG + residual add fused with the final rmsnorm. + norm_out = _sconv_add_norm( + pending[0], hidden_states, pending[1], self.norm, positions + )[0] + assert norm_out is not None + return norm_out + return self.norm(hidden_states) + + +class _TmlForCausalLMBase(nn.Module, SupportsPP, SupportsLoRA): + """Shared text-backbone causal-LM scaffolding for both entry classes.""" + + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={ + ".w13_dn": ".gate_up_proj", + ".w2_md": ".down_proj", + }, + orig_to_new_stacked={ + ".attn.wq_du.": (".attn.qkvr.", 0), + ".attn.wk_dv.": (".attn.qkvr.", 1), + ".attn.wv_dv.": (".attn.qkvr.", 2), + ".attn.wr_du.": (".attn.qkvr.", 3), + }, + orig_to_new_prefix={ + "model.llm.layers.": "model.layers.", + "model.llm.embed_norm": "model.embed_norm", + "model.llm.embed": "model.embed_tokens", + "model.llm.norm": "model.norm", + "model.llm.unembed": "lm_head", + "language_model.layers.": "model.layers.", + "language_model.lm_head.": "lm_head.", + }, + orig_to_new_suffix={ + # NVFP4 scale + ".w13_weight.scale": ".w13_weight_scale", + ".w13_weight.scale2": ".w13_weight_scale_2", + ".w2_weight.scale": ".w2_weight_scale", + ".w2_weight.scale2": ".w2_weight_scale_2", + }, + ) + # Quark uses this mapping when resolving quantization exclusions for the + # four checkpoint attention projections fused into qkvr. + packed_modules_mapping = { + "qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"], + "w13": ["w1", "w3"], + } + embedding_modules = { + "lm_head": "output_embeddings", + } + + def _build( + self, + vllm_config: VllmConfig, + text_config: InklingModelConfig, + prefix: str, + ) -> None: + quant_config = vllm_config.quant_config + self.config = text_config + # ROCm checkpoints use Quark OCP MXFP4. The global Quark config is + # passed into each routed MoE and its exclusion list keeps the rest of + # Inkling in bf16. + # Read by the MRV2 runner to publish per-request short-conv metadata. + # Short convolution is intrinsic to Inkling, so this is always set. + self.uses_sconv = True + self.model = InklingModel( + config=text_config, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "model"), + ) + self.lm_head = ParallelLMHead( + text_config.padded_vocab_size, + text_config.hidden_size, + org_num_embeddings=text_config.padded_vocab_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + self.logits_processor = InklingLogitsProcessor( + text_config.padded_vocab_size, + org_vocab_size=text_config.vocab_size, + soft_cap=text_config.final_logit_softcapping, + logits_mup_width_multiplier=text_config.logits_mup_width_multiplier, + ) + self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.model.make_empty_intermediate_tensors + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, + ) -> torch.Tensor | IntermediateTensors: + return self.model( + input_ids, + positions, + intermediate_tensors, + inputs_embeds, + ) + + def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: + return self.logits_processor(self.lm_head, hidden_states) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + return _load_inkling_weights(self, weights, self.config) + + +class InklingForCausalLM(_TmlForCausalLMBase): + """Text-only entry point (``inkling_model`` checkpoints).""" + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + self._build(vllm_config, vllm_config.model_config.hf_config, prefix) + + +@MULTIMODAL_REGISTRY.register_processor( + InklingMultiModalProcessor, + info=InklingProcessingInfo, + dummy_inputs=InklingDummyInputsBuilder, +) +class InklingForConditionalGeneration(_TmlForCausalLMBase, SupportsMultiModal): + """Top-level (multimodal) entry point. + + Builds the vision + audio towers on top of the shared text backbone. Inkling has + NO cross-modal fusion (the vision tower emits one token per patch, the audio + tower one token per frame), so generation reuses the inherited backbone + ``forward`` / ``compute_logits`` (the latter already applies muP) and this + class only adds multimodal embedding + merge. + """ + + hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_prefix={ + "model.audio.": "audio.", + "model.visual.": "visual.vision_encoder.", + }, + ) + + @classmethod + def get_placeholder_str(cls, modality: str, i: int) -> str | None: + if modality.startswith("image"): + return "<|content_image|>" + if modality.startswith("audio"): + return "<|content_audio_input|>" + raise ValueError("Only image or audio modality is supported") + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + config: InklingMMConfig = vllm_config.model_config.hf_config + + self.visual = ( + InklingVision(config.vision_config, prefix=maybe_prefix(prefix, "visual")) + if inkling_vision_enabled(config) + else None + ) + self.audio = ( + InklingAudio(config.audio_config, prefix=maybe_prefix(prefix, "audio")) + if inkling_audio_enabled(config) + else None + ) + + self._build(vllm_config, config.text_config, prefix) + + # -- multimodal embedding ------------------------------------------- + + def _process_image_input( + self, pixel_values: Any, num_patches: Any + ) -> tuple[torch.Tensor, ...]: + assert self.visual is not None + # pixel_values is a list (per item) of [P_i, 2, P, P, 3] tensors, + # or a single concatenated tensor. Normalize to a flat batch, run the + # tower once, then split back per item. + if isinstance(pixel_values, (list, tuple)): + if not pixel_values: + return () + sizes = [int(p.shape[0]) for p in pixel_values] + patches = torch.cat(list(pixel_values), dim=0) + else: + patches = pixel_values + sizes = self._sizes_from(num_patches, patches.shape[0]) + + patches = patches.to(device=self.visual.device, dtype=self.visual.dtype) + embeds = self.visual(patches) # [total_patches, D] + return tuple(embeds.split(sizes)) + + def _process_audio_input( + self, input_audio_features: Any, num_audio_tokens: Any + ) -> tuple[torch.Tensor, ...]: + assert self.audio is not None + if isinstance(input_audio_features, (list, tuple)): + if not input_audio_features: + return () + sizes = [int(d.shape[0]) for d in input_audio_features] + dmel = torch.cat(list(input_audio_features), dim=0) + else: + dmel = input_audio_features + sizes = self._sizes_from(num_audio_tokens, dmel.shape[0]) + + dmel = dmel.to(device=self.audio.device) + embeds = self.audio(dmel) # [total_frames, D] + return tuple(embeds.split(sizes)) + + @staticmethod + def _sizes_from(counts: Any, total: int) -> list[int]: + if counts is None: + return [total] + if isinstance(counts, torch.Tensor): + return [int(c) for c in counts.flatten().tolist()] + if isinstance(counts, (list, tuple)): + flat: list[int] = [] + for c in counts: + flat.append(int(c.item()) if isinstance(c, torch.Tensor) else int(c)) + return flat + return [int(counts)] + + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: + # Iterate modalities in a stable order so the returned per-item tensors + # line up with their appearance order; the positional merge in + # embed_input_ids handles actual placement. + pixel_values = kwargs.get("pixel_values") + num_patches = kwargs.get("num_patches") + input_audio_features = kwargs.get("input_audio_features") + num_audio_tokens = kwargs.get("num_audio_tokens") + + embeddings: tuple[torch.Tensor, ...] = () + if pixel_values is not None and self.visual is not None: + embeddings += self._process_image_input(pixel_values, num_patches) + if input_audio_features is not None and self.audio is not None: + embeddings += self._process_audio_input( + input_audio_features, num_audio_tokens + ) + return embeddings + + def embed_input_ids( + self, + input_ids: torch.Tensor, + multimodal_embeddings: MultiModalEmbeddings | None = None, + *, + is_multimodal: torch.Tensor | None = None, + ) -> torch.Tensor: + # Override the base's 1-arg embed_input_ids: the runner calls this 3-arg + # signature for multimodal models. Text embeddings come from the shared + # backbone (which applies embed_norm); MM embeddings are scattered in. + from vllm.model_executor.models.utils import _merge_multimodal_embeddings + + # Placeholder ids use unused vocabulary slots and these positions are + # overwritten by MM embeds below. + inputs_embeds = self.model.embed_input_ids(input_ids) + if multimodal_embeddings is None or len(multimodal_embeddings) == 0: + return inputs_embeds + assert is_multimodal is not None + return _merge_multimodal_embeddings( + inputs_embeds=inputs_embeds, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + + def get_language_model(self) -> nn.Module: + # This class IS the causal LM (the towers are side branches), so the + # language model is self — callers expect a module exposing ``.model`` + # / ``.lm_head`` (e.g. the MTP/eagle loader shares embeddings via + # ``get_language_model().model.embed_tokens``). + return self + + +# =========================================================================== +# Weight loading +# =========================================================================== + + +_MOE_EXPERT_WEIGHT_RE = re.compile( + r"^(?P.*\.mlp)\.(?P(?:shared_)?experts\..+)$" +) + + +def _is_peft_adapter_weight(name: str) -> bool: + return ".lora_A." in name or ".lora_B." in name + + +def _load_inkling_weights( + module: nn.Module, + weights: Iterable[tuple[str, torch.Tensor]], + config: InklingModelConfig, +) -> set[str]: + moe_modules = { + name: mod for name, mod in module.named_modules() if isinstance(mod, InklingMoE) + } + loaded: set[str] = set() + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + local_ids = set(config.local_layer_ids) + + def _iter_loadable_weights() -> Iterable[tuple[str, torch.Tensor]]: + for name, weight in module.hf_to_vllm_mapper.apply(weights): + # LightSeek's MXFP4 conversion bundles a quantized copy of the + # standalone PEFT adapter in the base safetensors index. These are + # not base-model parameters; adapters remain opt-in at serving. + if _is_peft_adapter_weight(name): + continue + shard_id = getattr(weight, "shard_id", None) + # Replicate K/V conv-free GQA heads when tp_size > num_kv_heads. + if ( + shard_id in (1, 2) + and name.endswith(".attn.qkvr.weight") + and weight.shape[0] > 0 + ): + lid = _layer_id(name) + if lid is not None: + is_local = lid in local_ids + n_kv = ( + config.swa_num_key_value_heads + if is_local + else config.num_key_value_heads + ) + head_dim = config.swa_head_dim if is_local else config.head_dim + if tp_size > n_kv and weight.shape[0] == n_kv * head_dim: + kv_idx = (tp_rank * n_kv) // tp_size + weight = weight.narrow(0, kv_idx * head_dim, head_dim) + weight.shard_id = shard_id + + # MoE expert tensors (fused stacked, routed + shared sink): translate + # the checkpoint layout to per-expert FusedMoE loads. + moe_match = _MOE_EXPERT_WEIGHT_RE.match(name) + if moe_match is not None and moe_match.group("mlp") in moe_modules: + moe = moe_modules[moe_match.group("mlp")] + for rel in moe.load_expert_weight(moe_match.group("rest"), weight): + loaded.add(f"{moe_match.group('mlp')}.{rel}") + continue + + yield name, weight + + loader = AutoWeightsLoader(module, skip_prefixes=["model.mtp."]) + loaded |= loader.load_weights(_iter_loadable_weights()) + + # Post-load MoE fixups (default input scales, zeroed EP-padding experts). + for moe_name, moe in moe_modules.items(): + for rel in moe.finalize_load(): + loaded.add(f"{moe_name}.{rel}") + return loaded + + +EntryClass = [InklingForCausalLM, InklingForConditionalGeneration] diff --git a/vllm/models/inkling/amd/moe.py b/vllm/models/inkling/amd/moe.py new file mode 100644 index 0000000000..90b00a3e5a --- /dev/null +++ b/vllm/models/inkling/amd/moe.py @@ -0,0 +1,693 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling mixture-of-experts on vLLM's FusedMoE abstraction. + +Overfit to the served checkpoint: sigmoid gate (+ selection bias) top-k over +the routed experts, log-sigmoid renormalization over the k routed + S shared +"sink" logits, scaled by route_scale * global_scale. The routed top-k goes +through vLLM's FusedMoE (which handles TP/EP); the sink experts run in +:class:`InklingSinkExperts` -- replicated across EP ranks (every token +activates every sink) and always bf16 (the checkpoint excludes every +``shared_experts`` from quantization). + +MXFP4 routed experts reuse vLLM's Quark OCP MXFP4 fused-MoE method; excluded +(bf16) layers fall back to the unquantized method. The checkpoint's fused +stacked tensors (interleaved gate/up rows, ``.scale`` / ``.scale2`` / +``.input_amax`` aux tensors) are translated to the standard per-expert loads +in :meth:`InklingMoE.load_expert_weight`. +""" + +from __future__ import annotations + +import math +from typing import TYPE_CHECKING + +import torch +from torch import nn +from torch.nn.parameter import Parameter + +import vllm.envs as envs +from vllm.config import get_current_vllm_config +from vllm.distributed import ( + get_dp_group, + get_pcp_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16 +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.utils import set_weight_attrs +from vllm.platforms import current_platform +from vllm.triton_utils import tl, tldevice, triton +from vllm.utils.multi_stream_utils import maybe_execute_in_parallel +from vllm.utils.torch_utils import aux_stream + +from ..configs import InklingModelConfig + +if TYPE_CHECKING: + from vllm.model_executor.layers.fused_moe.routed_experts import ( + RoutedExperts, + ) + +# --------------------------------------------------------------------------- +# Gate / expert selection +# --------------------------------------------------------------------------- + +_INKLING_LL_BF16_MAX_TOKENS = 64 +_MXFP4_INPUT_SCALE_DENOMINATOR = torch.finfo(torch.float8_e4m3fn).max * 6.0 + + +def _linear_with_fp32_out(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + leading = list(x.shape[:-1]) + flat = x.flatten(0, -2) + if ( + flat.shape[0] <= _INKLING_LL_BF16_MAX_TOKENS + and flat.dtype == torch.bfloat16 + and weight.dtype == torch.bfloat16 + and flat.is_cuda + and flat.is_contiguous() + and weight.is_contiguous() + and flat.shape[1] % 8 == 0 + and current_platform.has_device_capability(90) + and ll_bf16.is_available() + ): + out = ll_bf16.ll_bf16_gemm(flat, weight) + else: + out = torch.mm(flat, weight.T, out_dtype=torch.float32) + return out.view(*leading, weight.shape[0]) + + +@triton.jit(do_not_specialize=["T", "route_scale"]) +def _inkling_gate_select_kernel( + logits_ptr, # [T, G] fp32 gate logits (stride_logits_0 may include pad) + bias_ptr, # [R] fp32 selection bias (or 0 ptr if HAS_BIAS=False) + global_scale_ptr, # [1] fp32 (or unused if HAS_GSCALE=False) + ids_ptr, # [T, K + S] int32 out: selected expert ids + weights_ptr, # [T, K + S] fp32 out: renormalized weights + route_scale, + T, + G: tl.constexpr, # total gate experts (routed + shared) + stride_logits_0, + R: tl.constexpr, # routed experts + K: tl.constexpr, # top-k routed + S: tl.constexpr, # shared (sink) experts + HAS_BIAS: tl.constexpr, + HAS_GSCALE: tl.constexpr, + BLOCK_G: tl.constexpr, +): + pid = tl.program_id(0).to(tl.int64) + if pid >= T: + return + offs = tl.arange(0, BLOCK_G) + mask_r = offs < R + logits = tl.load( + logits_ptr + pid * stride_logits_0 + offs, + mask=offs < G, + other=float("-inf"), + ).to(tl.float32) + + # Selection scores: sigmoid(routed logits) (+ bias), non-routed lanes -inf. + sel = tl.where(mask_r, tl.sigmoid(logits), float("-inf")) + if HAS_BIAS: + bias = tl.load(bias_ptr + offs, mask=mask_r, other=0.0).to(tl.float32) + sel = tl.where(mask_r, sel + bias, float("-inf")) + + scale = route_scale + if HAS_GSCALE: + scale = scale * tl.load(global_scale_ptr).to(tl.float32) + + # Iterative top-K (K is small); argmax tie-breaks to the lowest index + # (stable ordering). + A: tl.constexpr = K + S + offs_a = tl.arange(0, A) + top_ids = tl.zeros([A], dtype=tl.int32) + active = tl.zeros([A], dtype=tl.float32) + for kk in tl.static_range(K): + idx = tl.argmax(sel, axis=0).to(tl.int32) + raw = tl.max(tl.where(offs == idx, logits, float("-inf")), axis=0) + top_ids = tl.where(offs_a == kk, idx, top_ids) + active = tl.where(offs_a == kk, raw, active) + sel = tl.where(offs == idx, float("-inf"), sel) + if S > 0: + # Shared sink logits sit at the tail of the gate output; their expert + # ids continue after the routed range (R + j). + for jj in tl.static_range(S): + raw = tl.max(tl.where(offs == R + jj, logits, float("-inf")), axis=0) + top_ids = tl.where(offs_a == K + jj, tl.full([], R + jj, tl.int32), top_ids) + active = tl.where(offs_a == K + jj, raw, active) + + # Log-sigmoid renormalization over the K + S active logits. + abs_l = tl.abs(active) + min_l = tl.minimum(active, 0.0) + log_probs = min_l - tldevice.log1p(tldevice.exp(-abs_l)) + max_lp = tl.max(log_probs, axis=0) + exp_shifted = tldevice.exp(log_probs - max_lp) + sum_exp = tl.sum(exp_shifted, axis=0) + weights = exp_shifted / sum_exp * scale + + tl.store(ids_ptr + pid * A + offs_a, top_ids) + tl.store(weights_ptr + pid * A + offs_a, weights) + + +def inkling_gate_select( + logits: torch.Tensor, # [T, >=G] fp32 (rows may carry GEMM padding) + n_gate_experts: int, + n_routed_experts: int, + topk: int, + n_shared_experts: int, + bias: torch.Tensor | None, + route_scale: float, + global_scale: torch.Tensor | None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Sigmoid + bias + top-k + log-sigmoid renorm; returns (weights, ids).""" + assert logits.dtype == torch.float32 + tokens = logits.shape[0] + active = topk + n_shared_experts + topk_ids = torch.empty((tokens, active), dtype=torch.int32, device=logits.device) + topk_weights = torch.empty( + (tokens, active), dtype=torch.float32, device=logits.device + ) + if tokens == 0: + return topk_weights, topk_ids + _inkling_gate_select_kernel[(tokens,)]( + logits, + bias if bias is not None else logits, + global_scale if global_scale is not None else logits, + topk_ids, + topk_weights, + route_scale, + tokens, + n_gate_experts, + logits.stride(0), + n_routed_experts, + topk, + n_shared_experts, + HAS_BIAS=bias is not None, + HAS_GSCALE=global_scale is not None, + BLOCK_G=triton.next_power_of_2(n_gate_experts), + ) + return topk_weights, topk_ids + + +class InklingGate(nn.Module): + """Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and + global scale (the served checkpoint's only configuration).""" + + def __init__( + self, + d_model: int, + n_routed_experts: int, + n_shared_experts: int, + experts_per_token: int, + route_scale: float, + *, + use_global_scale: bool = False, + use_gate_bias: bool = False, + ) -> None: + super().__init__() + self.n_routed_experts = n_routed_experts + self.n_shared_experts = n_shared_experts + self.n_total_experts = n_routed_experts + n_shared_experts + self.topk = experts_per_token + self.route_scale = route_scale + + padded_experts = self.n_total_experts + (-self.n_total_experts) % 8 + self.weight = Parameter( + torch.empty(padded_experts, d_model), requires_grad=False + ) + set_weight_attrs(self.weight, {"weight_loader": self._load_weight}) + self.global_scale: Parameter | None + if use_global_scale: + self.global_scale = Parameter( + torch.empty(1, dtype=torch.float32), requires_grad=False + ) + else: + self.global_scale = None + self.bias: Parameter | None + if use_gate_bias: + self.bias = Parameter( + torch.empty(n_routed_experts, dtype=torch.float32), + requires_grad=False, + ) + else: + self.bias = None + + @staticmethod + def _load_weight(param: Parameter, loaded_weight: torch.Tensor) -> None: + param.data.zero_() + param.data[: loaded_weight.shape[0]].copy_(loaded_weight) + + def compute_logits(self, x: torch.Tensor) -> torch.Tensor: + """fp32 gate logits [T, n_total_experts + pad] (pad columns are junk).""" + return _linear_with_fp32_out(x, self.weight) + + def select_experts( + self, gating_output: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """Full selection: (weights, ids) of [T, K + S]. The first K entries + are the routed top-k; the S trailing entries are the sink gammas.""" + return inkling_gate_select( + gating_output, + self.n_total_experts, + self.n_routed_experts, + self.topk, + self.n_shared_experts, + self.bias, + self.route_scale, + self.global_scale, + ) + + +# --------------------------------------------------------------------------- +# MoE layer +# --------------------------------------------------------------------------- + + +def _inkling_moe_ep_size() -> int: + """EP size the FusedMoE layer will run with (mirrors + FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when + expert parallelism is enabled).""" + parallel_config = get_current_vllm_config().parallel_config + if not parallel_config.enable_expert_parallel: + return 1 + world = ( + get_tensor_model_parallel_world_size() + * get_dp_group().world_size + * get_pcp_group().world_size + ) + return world if world > 1 else 1 + + +class InklingSinkExperts(nn.Module): + """Shared "sink" experts with per-token gammas, in bf16. + + Replicated across EP ranks (every token activates every sink, so + EP-sharding them would hotspot the owning rank) and TP-sharded on the + intermediate dim so the output remains a TP-partial sum like the routed + output. The sinks are always bf16 (the checkpoint excludes every + ``shared_experts`` from quantization): the experts concatenate into two + plain dense GEMMs with the fused sink epilogue between them. + """ + + def __init__( + self, n_experts: int, d_model: int, d_mlp: int, *, prefix: str = "" + ) -> None: + super().__init__() + self.n_experts = n_experts + tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + intermediate_pp = d_mlp // tp_size + self.w13_weight = Parameter( + torch.empty(n_experts, 2 * intermediate_pp, d_model), + requires_grad=False, + ) + self.w2_weight = Parameter( + torch.empty(d_model, n_experts * intermediate_pp), + requires_grad=False, + ) + self._unit: torch.Tensor | None = None + + def load_weight(self, key: str, weight: torch.Tensor) -> list[str]: + """Load one checkpoint sink tensor (stacked over the S experts).""" + if key == "w13_weight": + if weight.shape != self.w13_weight.shape: + shard = self.w13_weight.shape[1] + weight = weight.narrow(1, self.tp_rank * shard, shard) + self.w13_weight.data.copy_(weight) + return [key] + + assert key == "w2_weight" + shard = self.w2_weight.shape[1] // self.n_experts + shard_start = 0 if weight.shape[2] == shard else self.tp_rank * shard + for expert_idx, expert_weight in enumerate(weight): + local_weight = expert_weight.narrow(1, shard_start, shard) + start = expert_idx * shard + self.w2_weight.data[:, start : start + shard].copy_(local_weight) + return [key] + + def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor: + """``sum_e gammas[:, e] * MLP_e(x)`` (TP-partial along d_mlp).""" + from .ops import sink_silu_mul_epilogue + + # One GEMM over the experts' stacked w13 (a view), fused epilogue, + # then one GEMM whose K-reduction over the K-concatenated w2 performs + # the expert sum. + if self._unit is None or self._unit.device != x.device: + self._unit = torch.ones( + self.n_experts, dtype=torch.float32, device=x.device + ) + raw = x @ self.w13_weight.view(-1, x.shape[-1]).T # (T, S*2F) + h = sink_silu_mul_epilogue( + raw, self._unit, gammas, self._unit, self.n_experts, x.dtype + ) + return h @ self.w2_weight.T # (T, D) + + +class InklingSinkExpertsLinear(nn.Module): + """LoRA-capable implementation of the Inkling sink experts.""" + + def __init__( + self, + n_experts: int, + d_model: int, + d_mlp: int, + *, + prefix: str = "", + ) -> None: + super().__init__() + from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + RowParallelLinear, + ) + + self.n_experts = n_experts + self.d_mlp = d_mlp + total = n_experts * d_mlp + self.w13 = MergedColumnParallelLinear( + input_size=d_model, + output_sizes=[total, total], + bias=False, + prefix=f"{prefix}.w13", + ) + self.w2 = RowParallelLinear( + input_size=total, + output_size=d_model, + bias=False, + reduce_results=False, + prefix=f"{prefix}.w2", + ) + self._w2_input_pp = self.w2.input_size_per_partition + self._col_expert: torch.Tensor | None = None + + def _gamma_expand(self, gammas: torch.Tensor) -> torch.Tensor: + if self._col_expert is None or self._col_expert.device != gammas.device: + local = self._w2_input_pp + start = get_tensor_model_parallel_rank() * local + cols = torch.arange(start, start + local, device=gammas.device) + self._col_expert = (cols // self.d_mlp).long() + return gammas[:, self._col_expert] + + def load_weight(self, key: str, weight: torch.Tensor) -> list[str]: + if key == "w13_weight": + d_model = weight.shape[-1] + gate = weight[:, 0::2, :].reshape(-1, d_model).contiguous() + up = weight[:, 1::2, :].reshape(-1, d_model).contiguous() + self.w13.weight_loader(self.w13.weight, gate, 0) + self.w13.weight_loader(self.w13.weight, up, 1) + return ["w13.weight"] + w = weight.permute(1, 0, 2).reshape(weight.shape[1], -1).contiguous() + self.w2.weight_loader(self.w2.weight, w) + return ["w2.weight"] + + def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.w13(x) + gate, up = gate_up.chunk(2, dim=-1) + hidden_states = torch.nn.functional.silu(gate) * up + hidden_states = (hidden_states * self._gamma_expand(gammas)).to(x.dtype) + output, _ = self.w2(hidden_states) + return output + + +class InklingMoE(nn.Module): + def __init__( + self, + config: InklingModelConfig, + *, + prefix: str = "", + quant_config: QuantizationConfig | None = None, + ) -> None: + super().__init__() + # Overfit to the served checkpoint: sigmoid gate renormalized after + # top-k, shared sink experts, interleaved gate/up checkpoint rows. + assert config.gate_activation == "sigmoid" and config.norm_after_topk + assert config.n_shared_experts > 0 and config.shared_expert_sink + assert config.inference_moe_w13_interleaved + n_routed = config.n_routed_experts + n_shared = config.n_shared_experts + self.n_routed_experts = n_routed + self.gate = InklingGate( + d_model=config.hidden_size, + n_routed_experts=n_routed, + n_shared_experts=n_shared, + experts_per_token=config.num_experts_per_tok, + route_scale=config.route_scale, + use_global_scale=config.use_global_scale, + use_gate_bias=config.use_gate_bias, + ) + + # TRTLLM MoE kernels assume equal, contiguous per-rank expert slabs + # (local_expert_offset = ep_rank * local_num_experts), so pad the + # expert count to a multiple of the EP size. A no-op for the usual + # power-of-two EP sizes (n_routed is a power of two). + num_experts = n_routed + (-n_routed) % _inkling_moe_ep_size() + + # The released MXFP4 checkpoint keeps the first routed-expert layer + # in bf16 and lists its two expert weights explicitly in Quark's + # exclusion list. FusedMoE asks for a quant method at the module + # prefix, so an exact weight exclusion would otherwise be missed and + # the bf16 tensors would be loaded into MXFP4 parameters. + routed_quant_config = quant_config + if quant_config is not None: + excluded = set(getattr(quant_config, "quant_config", {}).get("exclude", ())) + expert_prefix = f"{prefix}.experts" + routed_weights = { + f"{expert_prefix}.w13_weight", + f"{expert_prefix}.w2_weight", + } + if routed_weights <= excluded: + routed_quant_config = None + + self.experts = FusedMoE( + num_experts=num_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + renormalize=False, + quant_config=routed_quant_config, + prefix=f"{prefix}.experts", + custom_routing_function=self._select_routed, + router_logits_dtype=torch.float32, + activation="silu", + ) + # The decoder layer reduce-scatters the MoE delta into the sconv + # stream itself (RS -> shard sconv -> AG); the runner must return the + # per-rank partial sum instead of all-reducing. + self.experts.moe_config.skip_final_all_reduce = True + + self._routed_sel: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None + sink_experts_cls = ( + InklingSinkExpertsLinear + if get_current_vllm_config().lora_config is not None + else InklingSinkExperts + ) + self.sink_experts = sink_experts_cls( + n_experts=n_shared, + d_model=config.hidden_size, + d_mlp=config.intermediate_size, + prefix=f"{prefix}.shared_experts", + ) + + # Sink chain overlaps the routed MoE call on the aux stream for + # decode-sized batches (same pattern as the runner's SharedExperts + # multi-stream overlap). The routed GEMM runs on the default stream and + # the sink chain on the aux stream, joined via these two events by + # ``maybe_execute_in_parallel``. + self._sink_stream: torch.cuda.Stream | None = aux_stream() + self._sink_events = (torch.cuda.Event(), torch.cuda.Event()) + + def _select_routed( + self, + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + ) -> tuple[torch.Tensor, torch.Tensor]: + """FusedMoE ``custom_routing_function``: the routed top-k slice of the + full (routed + sink) selection. + + forward() stashes its selection (keyed by logits identity) so the + gate select runs once per layer; the fallback covers paths where the + runner re-derives the logits (e.g. naive DP dispatch). + """ + del hidden_states, renormalize + assert topk == self.gate.topk + cached = self._routed_sel + self._routed_sel = None + if cached is not None and cached[0] is gating_output: + return cached[1], cached[2] + weights, ids = self.gate.select_experts(gating_output) + return weights[:, :topk].contiguous(), ids[:, :topk].contiguous() + + def forward(self, x: torch.Tensor) -> torch.Tensor | None: + router_logits = self.gate.compute_logits(x) + num_tokens = x.shape[0] + # One gate select per layer: the routed slice is stashed for the + # routing function inside the FusedMoE op; the sink gammas are the + # trailing columns. + k = self.gate.topk + weights, ids = self.gate.select_experts(router_logits) + self._routed_sel = ( + router_logits, + weights[:, :k].contiguous(), + ids[:, :k].contiguous(), + ) + gammas = weights[:, k:] + + out, sink_out = maybe_execute_in_parallel( + lambda: self.experts(hidden_states=x, router_logits=router_logits), + lambda: self.sink_experts(x, gammas), + self._sink_events[0], + self._sink_events[1], + self._sink_stream + if num_tokens <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD + else None, + ) + self._routed_sel = None + + return out.add_(sink_out) + + # -- weight loading ---------------------------------------------------- + + def _local_expert_slots(self) -> dict[int, int]: + """Global expert id -> local slot for this rank's expert partition.""" + manager = self.experts.routed_experts.expert_map_manager + if manager.expert_map is None: + return {g: g for g in range(manager.global_num_experts)} + emap = manager.expert_map.tolist() + return {g: slot for g, slot in enumerate(emap) if slot >= 0} + + def load_expert_weight(self, name: str, weight: torch.Tensor) -> list[str]: + """Load one checkpoint expert tensor. + + ``name`` is relative to the mlp module: ``experts.`` (routed + stack) or ``shared_experts.shared_`` (sink experts). Returns the + loaded param names (relative to this module). + """ + if name.startswith("shared_experts."): + key = name.split(".", 1)[1].replace("shared_", "", 1) + return [ + f"sink_experts.{p}" for p in self.sink_experts.load_weight(key, weight) + ] + + experts: RoutedExperts = self.experts.routed_experts + key = name.split(".", 1)[1] + + # original_shape is unused by the vLLM serving layout. + if key.endswith(".original_shape"): + return [] + if key.endswith(".input_amax"): + projection = "w13" if key.startswith("w13") else "w2" + amax = float(weight.max()) + assert math.isfinite(amax) and amax > 0, ( + f"bad {projection} input_amax: {amax}" + ) + input_scale = getattr(experts, f"{projection}_input_scale") + input_scale.data.fill_(amax / _MXFP4_INPUT_SCALE_DENOMINATOR) + return [f"experts.routed_experts.{projection}_input_scale"] + + # Quark's OCP MXFP4 converter stores block scales as a two-dimensional + # tensor with the expert and projection-row dimensions flattened: + # w13: [E * 2I, H / 32], w2: [E * H, I / 32]. + # Restore the expert dimension before applying the EP/TP slicing used + # for both packed weights and scales. + if key.endswith("_weight_scale") and weight.ndim == 2: + if weight.shape[0] % self.n_routed_experts != 0: + raise ValueError( + f"cannot unflatten {name} with shape {tuple(weight.shape)} " + f"over {self.n_routed_experts} experts" + ) + weight = weight.view( + self.n_routed_experts, + weight.shape[0] // self.n_routed_experts, + weight.shape[1], + ) + + param = getattr(experts, key) + slots = self._local_expert_slots() + gids = sorted(slots) + lids = [slots[g] for g in gids] + tp_rank = experts.moe_config.moe_parallel_config.tp_rank + tp_size = experts.moe_config.moe_parallel_config.tp_size + + if key.endswith("_scale_2"): + # Per-expert scalars, vectorized over the local experts. The + # fused w13 param carries one slot per gate/up half. + vals = weight[gids].float().to(param.device) + param.data[lids] = vals[:, None] if param.data.ndim == 2 else vals + elif key.startswith("w13"): + # Checkpoint w13 rows are interleaved [g0, u0, g1, u1, ...]; the + # fused param layout is [w1(gate); w3(up)]. The TP-local rows form + # one contiguous slab of the interleaved tensor, so upload just + # that slab (a single bounded synchronous H2D; pre-uploading whole + # untrimmed tensors pins the mmap pages of the entire checkpoint + # and OOMs the host) and de-interleave on device. + dst_half = param.shape[1] // 2 + if weight.shape[1] % (2 * tp_size) != 0: + raise ValueError( + f"cannot TP-shard {name} with shape {tuple(weight.shape)} " + f"over {tp_size} ranks" + ) + logical_half = weight.shape[1] // (2 * tp_size) + if logical_half > dst_half: + raise ValueError( + f"checkpoint shard for {name} has {logical_half} rows per " + f"gate/up half, but destination only has {dst_half}" + ) + for gid, lid in slots.items(): + slab = weight[gid].narrow( + 0, + tp_rank * 2 * logical_half, + 2 * logical_half, + ) + slab = slab.to(param.device) + param.data[lid, :logical_half].copy_(slab[0::2]) + param.data[ + lid, + dst_half : dst_half + logical_half, + ].copy_(slab[1::2]) + else: + # w2: shard the packed intermediate (last) dim. AITER rounds the + # destination intermediate width to 256, so derive the logical TP + # slice from the checkpoint and leave the destination tail at its + # initialized padding value (zero for weights, one for scales). + if weight.shape[2] % tp_size != 0: + raise ValueError( + f"cannot TP-shard {name} with shape {tuple(weight.shape)} " + f"over {tp_size} ranks" + ) + shard = weight.shape[2] // tp_size + if shard > param.shape[2]: + raise ValueError( + f"checkpoint shard for {name} has width {shard}, but " + f"destination only has {param.shape[2]}" + ) + for gid, lid in slots.items(): + param.data[lid, :, :shard].copy_( + weight[gid].narrow(1, tp_rank * shard, shard) + ) + return [f"experts.routed_experts.{key}"] + + def finalize_load(self) -> list[str]: + """Post-load fixups for zeroed padding experts.""" + experts = self.experts.routed_experts + out: list[str] = [] + # Zero the EP-alignment padding experts (if any) so their + # (never-routed) slots hold defined values. + slots = self._local_expert_slots() + for gid in range(self.n_routed_experts, experts.global_num_experts): + lid = slots.get(gid) + if lid is None: + continue + for pname in ( + "w13_weight", + "w2_weight", + "w13_weight_scale", + "w2_weight_scale", + "w13_weight_scale_2", + "w2_weight_scale_2", + ): + p = getattr(experts, pname, None) + if p is not None: + p.data[lid].zero_() + return out diff --git a/vllm/models/inkling/amd/mtp.py b/vllm/models/inkling/amd/mtp.py new file mode 100644 index 0000000000..34cde020ed --- /dev/null +++ b/vllm/models/inkling/amd/mtp.py @@ -0,0 +1,410 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling MTP (Multi-Token Prediction) draft model (NVIDIA). + +Implements the first MTP depth from the reference ``mtp_model.py`` shipped with +the checkpoint. It owns ``hidden_norm`` / ``embed_norm`` RMSNorms, a ``2H -> H`` +input projection, and a full Inkling transformer block with a dense bf16 MLP. + +The draft shares the target's token embedding table and LM head +(``load_eagle_model`` wires those references) and applies the backbone +``embed_norm`` on top: the depth layers were trained on the same normed +embeddings the backbone consumes (their own ``embed_norm`` weights are +near-identity trims, unlike the backbone's whitening ``embed_norm``). +""" + +from __future__ import annotations + +from collections.abc import Iterable + +import regex as re +import torch +from torch import nn + +from vllm.config import VllmConfig +from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.utils import maybe_prefix +from vllm.sequence import IntermediateTensors + +from ..configs import InklingModelConfig +from .layernorm import InklingRMSNorm +from .model import InklingDecoderLayer, InklingReplicatedEmbedding +from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm + +# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr. +# Mirrors the backbone's hf_to_vllm_mapper.orig_to_new_stacked; kept as a +# local (pname, wname, shard) list since the MTP loader remaps by hand. +_ATTENTION_PARAMS_MAPPING = [ + ("qkvr", "wq_du", 0), + ("qkvr", "wk_dv", 1), + ("qkvr", "wv_dv", 2), + ("qkvr", "wr_du", 3), +] + + +def _mtp_depth_from_name(name: str) -> int | None: + m = re.search(r"\.mtp\.layers\.(\d+)\.", name) + return int(m.group(1)) if m else None + + +class InklingMTPDepthLayer(nn.Module): + """One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block.""" + + def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None: + super().__init__() + self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.input_proj = ReplicatedLinear( + config.hidden_size * 2, + config.hidden_size, + bias=False, + return_bias=False, + prefix=f"{prefix}.input_proj", + ) + # A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs + # full attention (the swa_* head config and sliding_window window) to + # match this depth's checkpoint transformer_block weights. + self.transformer_block = InklingDecoderLayer( + config, + layer_id=0, + is_local=is_local, + quant_config=None, + prefix=f"{prefix}.transformer_block", + force_dense_mlp=True, + ) + + def forward(self, combined: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + # ``combined`` is the fused-normed [rmsnorm(hidden) | embed_norm(emb)] + # input, built by InklingMultiTokenPredictor.fused_input_cat in one launch. + hidden = self.input_proj(combined) + # The short conv self-fetches its paged SWA-cache metadata from the + # forward context (via its conv_owner prefix); no conv_meta to thread. + return self.transformer_block(positions, hidden) + + +class InklingMultiTokenPredictor(nn.Module): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + assert vllm_config.speculative_config is not None + config: InklingModelConfig = ( + vllm_config.speculative_config.draft_model_config.hf_config + ) + self.config = config + if vllm_config.speculative_config.num_speculative_tokens != 1: + raise ValueError( + "Inkling MTP currently supports exactly one speculative token" + ) + self.chain_hidden_post_norm = config.chain_hidden_post_norm + local_ids = set(config.local_layer_ids) + self.layers = nn.ModuleDict( + {"0": InklingMTPDepthLayer(config, f"{prefix}.layers.0", 0 in local_ids)} + ) + self.chain_norm = ( + InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + if self.chain_hidden_post_norm + else None + ) + # The target's raw token embedding (pre embed_norm), attached by + # load_eagle_model. Never materialized here: building our own + # replicated copy would transiently double the 2.3 GiB table. + self.embed_tokens: InklingReplicatedEmbedding = None # type: ignore[assignment] + # The depth layers consume the *backbone-normed* embedding + # (embed_norm(embed(ids))), not the raw one: mtp embed_norm weights + # are near-identity (trained on already-normalized inputs), and + # feeding raw embeddings drops MTP1 acceptance from ~0.85 to ~0.70. + # Weight loaded from the target's embed_norm.weight; gated like the + # target's InklingModel.embed_norm. + self.backbone_embed_norm = ( + InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + if config.use_embed_norm + else None + ) + + def embed_input_ids( + self, + input_ids: torch.Tensor, + multimodal_embeddings: object | None = None, + *, + is_multimodal: torch.Tensor | None = None, + ) -> torch.Tensor: + """Draft-prefill embedding: fused gather + backbone embed_norm, then + the target's tower embeddings scattered in unnormed (the backbone + convention — MM embeds are merged after embed_norm).""" + norm = self.backbone_embed_norm + embeds = embed_rmsnorm( + input_ids, + self.embed_tokens.weight, + norm.weight if norm is not None else None, + norm.variance_epsilon if norm is not None else 0.0, + ) + if multimodal_embeddings is None or len(multimodal_embeddings) == 0: # type: ignore[arg-type] + return embeds + from vllm.model_executor.models.utils import _merge_multimodal_embeddings + + assert is_multimodal is not None + return _merge_multimodal_embeddings( + inputs_embeds=embeds, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + + def fused_input_cat( + self, + layer: InklingMTPDepthLayer, + previous_hidden: torch.Tensor, + input_ids: torch.Tensor, + inputs_embeds: torch.Tensor | None, + ) -> torch.Tensor: + """The depth layer's [rmsnorm(hidden) | embed_norm(embed)] input in one + launch: embedding row gather + the backbone embed_norm + the depth + embed_norm chain on one side, hidden_norm on the other, written + straight into the cat buffer.""" + hidden_w = layer.hidden_norm.weight + embed_w = layer.embed_norm.weight + eps = layer.hidden_norm.variance_epsilon + if inputs_embeds is not None: + # Draft prefill with target-merged MM embeddings (already + # backbone-normed via embed_input_ids); only the depth embed_norm + # remains. + return embed_dual_rmsnorm_cat( + previous_hidden, hidden_w, embed_w, eps, embeds=inputs_embeds + ) + return embed_dual_rmsnorm_cat( + previous_hidden, + hidden_w, + embed_w, + eps, + input_ids=input_ids, + embed_table=self.embed_tokens.weight, + pre_norm_weight=( + self.backbone_embed_norm.weight + if self.backbone_embed_norm is not None + else None + ), + ) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + previous_hidden_states: torch.Tensor, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor: + # The draft's short conv is a paged SWA-cache layer (its conv_owner is + # auto-enumerated as a draft attention layer); its per-token metadata is + # built by the speculator's build_attn_metadata and read from the + # forward context, so nothing extra is threaded here. + if spec_step_idx != 0: + raise ValueError("Inkling MTP only supports spec_step_idx=0") + layer = self.layers["0"] + combined = self.fused_input_cat( + layer, previous_hidden_states, input_ids, inputs_embeds + ) + hidden = layer(combined, positions) + if self.chain_norm is not None: + hidden = self.chain_norm(hidden) + return hidden + + +class InklingMTP(nn.Module): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + assert vllm_config.speculative_config is not None + config: InklingModelConfig = ( + vllm_config.speculative_config.draft_model_config.hf_config + ) + self.config = config + self.model = InklingMultiTokenPredictor( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + # The target's (vocab-sharded) LM head, attached by load_eagle_model; + # never materialized here (same reasoning as model.embed_tokens). + self.lm_head: ParallelLMHead = None # type: ignore[assignment] + self.logits_processor = LogitsProcessor( + config.padded_vocab_size, + org_vocab_size=config.vocab_size, + soft_cap=config.final_logit_softcapping, + ) + self._logits_zero: torch.Tensor | None = None + + def embed_input_ids( + self, + input_ids: torch.Tensor, + multimodal_embeddings: object | None = None, + *, + is_multimodal: torch.Tensor | None = None, + ) -> torch.Tensor: + return self.model.embed_input_ids( + input_ids, multimodal_embeddings, is_multimodal=is_multimodal + ) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + hidden_states: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor: + return self.model( + input_ids, + positions, + hidden_states, + inputs_embeds, + spec_step_idx, + ) + + def compute_logits( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor | None: + # The MTP shares the base model's LM head, which is trained on + # ``hidden / mup``-scaled inputs, so apply the same mup scaling here + # for a matching logit scale — folded into the lm_head GEMM alpha + # (fp32 epilogue) like the target's compute_logits. (Argmax-invariant + # for greedy draft sampling, but it matters for the gumbel sampling + # distribution at temperature > 0.) + mup = self.config.logits_mup_width_multiplier + if not mup: + return self.logits_processor(self.lm_head, hidden_states) + assert self.logits_processor.soft_cap is None + assert self.logits_processor.scale == 1.0 + w = self.lm_head.weight + if self._logits_zero is None: + self._logits_zero = w.new_zeros(1) + logits = torch.addmm( + self._logits_zero, + hidden_states, + w.t(), + beta=0.0, + alpha=1.0 / mup, + ) + logits = self.logits_processor._gather_logits(logits) + if logits is not None: + logits = logits[..., : self.logits_processor.org_vocab_size] + return logits + + def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Greedy draft tokens via rank-local argmax + tiny (value, index) + reduction — no full-vocab logits all-gather. The muP divisor is a + positive scalar, so the argmax is invariant and the scaling is + skipped entirely.""" + return self.logits_processor.get_top_tokens(self.lm_head, hidden_states) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + return _load_inkling_mtp_weights(self, weights) + + +def _load_inkling_mtp_weights( + module: InklingMTP, + weights: Iterable[tuple[str, torch.Tensor]], +) -> set[str]: + """Load ``model.mtp.*`` weights into the MTP module. + + Checkpoint keys look like ``model.mtp.chain_norm.weight`` and + ``model.mtp.layers.{i}.{...}``. The transformer block reuses the backbone + layer's fused-projection layout, so we apply the same qkvr / gate_up / down + remapping as ``_load_inkling_weights``. Token embedding and LM head are shared + (provided by ``load_eagle_model``) and are not present in mtp.safetensors. + """ + # Per-depth attention is full or sliding-window (config.local_layer_ids); + # each depth's qkvr MergedColumnParallelLinear is built with the matching + # (swa_)num_key_value_heads, and its weight_loader handles the TP sharding. + # The sconv SWA cache pins tp_size <= num_key_value_heads, so tp never + # exceeds a layer's kv-head count and no GQA K/V replication is needed here. + params = dict(module.named_parameters()) + loaded: set[str] = set() + + def _load(name: str, weight: torch.Tensor, shard_id: object = None) -> bool: + param = params.get(name) + if param is None: + return False + loader = getattr(param, "weight_loader", default_weight_loader) + if shard_id is None: + if loader is default_weight_loader or param.shape == weight.shape: + default_weight_loader(param, weight) + else: + loader(param, weight) + else: + loader(param, weight, shard_id) # type: ignore[call-arg] + loaded.add(name) + return True + + for name, weight in weights: + depth = _mtp_depth_from_name(name) + # Token embedding and LM head are never materialized on the draft + # (no params to load into); load_eagle_model attaches the target's. + if name in ("model.llm.embed.weight", "model.llm.unembed.weight"): + continue + # The backbone embed_norm, applied to the shared embedding before the + # depth layers (see InklingMultiTokenPredictor.embed_input_ids). The + # per-depth mtp.layers.{i}.embed_norm keys carry ".mtp." and are loaded + # below. Only the shared backbone key routes here. + if name == "model.llm.embed_norm.weight": + _load("model.backbone_embed_norm.weight", weight) + continue + # Only consume the MTP weights; everything else belongs to the target. + if ".mtp." not in name: + continue + # Only the first checkpoint depth is used for MTP=1. + if depth is not None and depth != 0: + continue + # model.mtp.chain_norm.weight -> model.chain_norm.weight + # model.mtp.layers.{i}.X -> model.layers.{i}.X + original_name = name + name = name.replace(".mtp.layers.", ".layers.").replace( + ".mtp.chain_norm.", ".chain_norm." + ) + + if ".chain_norm." in name and module.model.chain_norm is None: + raise ValueError( + "Inkling checkpoint contains chain_norm weights but " + "chain_hidden_post_norm is disabled." + ) + + # Fused attention qkvr (wq_du/wk_dv/wv_dv/wr_du -> qkvr). + matched = False + for pname, wname, shard in _ATTENTION_PARAMS_MAPPING: + if f".attn.{wname}." in name: + mapped_name = name.replace(f".{wname}.", f".{pname}.") + if not _load(mapped_name, weight, shard): + raise ValueError(f"Unexpected Inkling MTP weight: {original_name}") + matched = True + break + if matched: + continue + + # Dense MLP fused gate/up + down. + if ".mlp.w13_dn.weight" in name: + loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight) + elif ".mlp.w2_md.weight" in name: + loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight) + else: + if name.endswith(".bias") and name not in params: + continue + loaded_weight = _load(name, weight) + if not loaded_weight: + raise ValueError(f"Unexpected Inkling MTP weight: {original_name}") + required = { + name + for name in params + if name.startswith("model.layers.") or name.startswith("model.chain_norm.") + } + if (missing := sorted(required - loaded)) and is_mtp_completeness_check_enabled(): + raise ValueError( + "Inkling MTP checkpoint is missing required parameters: " + + ", ".join(missing) + ) + return loaded + + +EntryClass = [InklingMTP] diff --git a/vllm/models/inkling/amd/ops/__init__.py b/vllm/models/inkling/amd/ops/__init__.py new file mode 100644 index 0000000000..3c6ea5fe8f --- /dev/null +++ b/vllm/models/inkling/amd/ops/__init__.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling kernels (NVIDIA). + +``rmsnorm`` / ``sconv`` import eagerly. The SwiGLU kernels and the FA4 +relative-attention wrapper are exposed lazily to keep this package's +import path lightweight. +""" + +from typing import TYPE_CHECKING + +from .norm import add_rmsnorm, rmsnorm +from .sconv import fused_sconv + +_LAZY_EXPORTS = { + "silu_and_mul_triton": "silu_and_mul", + "sink_silu_mul_epilogue": "silu_and_mul", + "inkling_fa4_rel_attention": "fa4_rel_attention", + "inkling_rel_attention_split_kv_decode": "rel_attention_decode", +} + +if TYPE_CHECKING: + from .fa4_rel_attention import inkling_fa4_rel_attention # noqa: F401 + from .silu_and_mul import ( # noqa: F401 + silu_and_mul_triton, + sink_silu_mul_epilogue, + ) + +__all__ = [ + "add_rmsnorm", + "rmsnorm", + "fused_sconv", + *sorted(_LAZY_EXPORTS), +] + + +def __getattr__(name: str): + module = _LAZY_EXPORTS.get(name) + if module is not None: + import importlib + + mod = importlib.import_module(f".{module}", __name__) + return getattr(mod, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/vllm/models/inkling/amd/ops/fa4_rel_attention.py b/vllm/models/inkling/amd/ops/fa4_rel_attention.py new file mode 100644 index 0000000000..d448ac8e2c --- /dev/null +++ b/vllm/models/inkling/amd/ops/fa4_rel_attention.py @@ -0,0 +1,438 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm paged attention with Inkling's query-dependent relative bias. + +The NVIDIA implementation uses the score-mod hook in tml-fa4. ROCm Flash +Attention and AITER do not expose an equivalent hook, so this module implements +the same operation directly in Triton. Query heads belonging to one KV head +are processed together and KV pages are gathered through vLLM's block table. +""" + +from __future__ import annotations + +import os +from typing import cast + +import torch + +from vllm.models.inkling.amd.ops.rel_attention_decode import ( + inkling_rel_attention_split_kv_decode, + use_split_kv_decode, +) +from vllm.platforms.rocm import on_gfx950 +from vllm.triton_utils import tl, triton + + +def bucket_max_seqlen_q(max_seqlen_q: int) -> int: + """Round the scheduling bound up to a power of two.""" + return 1 << max(0, max_seqlen_q - 1).bit_length() + + +def use_gfx950_gluon_decode( + *, max_query_len: int, page_size: int, head_dim: int +) -> bool: + """Use the vendored TokenSpeed CDNA4 decode kernel where it is supported.""" + return ( + os.getenv("INKLING_GFX950_GLUON", "1") == "1" + and on_gfx950() + and max_query_len == 1 + and page_size in (64, 128, 256) + and head_dim in (64, 128) + ) + + +def use_gfx950_gluon_extend( + *, + max_query_len: int, + max_kv_len: int, + page_size: int, + head_dim: int, + window_left: int, +) -> bool: + """Use Gluon only for the long full-attention extend regime it wins.""" + return ( + os.getenv("INKLING_GFX950_GLUON", "1") == "1" + and on_gfx950() + and max_query_len > 1 + and max_kv_len >= 8192 + and window_left < 0 + and page_size in (64, 128, 256) + and head_dim in (64, 128) + ) + + +def inkling_fa4_num_splits( + *, + is_local: bool, + batch_size: int, + max_query_len: int, + num_heads: int, + num_kv_heads: int, + max_kv_len: int, +) -> int: + """Keep the NVIDIA-facing split heuristic as API-compatible metadata. + + The ROCm Triton implementation performs online softmax in one program and + does not consume the result. Keeping this function unchanged avoids + platform-specific scheduling branches in :mod:`inkling.amd.attention`. + """ + if is_local: + return 1 + + q_rows = max_query_len * (num_heads // num_kv_heads) + q_tiles = (q_rows + 255) // 256 + base_ctas = batch_size * num_kv_heads * q_tiles + target_ctas = ( + 256 if q_tiles == 1 and batch_size == 1 else (128 if q_tiles == 1 else 64) + ) + max_splits = 128 + if q_tiles == 1 and batch_size == 1: + if num_kv_heads == 8: + max_splits = 16 + elif num_kv_heads == 4 or max_kv_len <= 8192: + max_splits = 32 + elif max_kv_len <= 65536: + max_splits = 64 + return max( + 1, + min(target_ctas // base_ctas, max_splits, (max_kv_len + 127) // 128), + ) + + +@triton.heuristics( + { + "BLOCK_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "BLOCK_H": lambda args: triton.next_power_of_2(args["gqa_group_size"]), + "BLOCK_QH": lambda args: ( + args["BLOCK_Q"] * triton.next_power_of_2(args["gqa_group_size"]) + ), + } +) +@triton.jit(do_not_specialize_on_alignment=["cache_seqlens", "cu_seqlens_q"]) +def _inkling_rel_attention_kernel( + q_ptr, # [total_q, Hq, D] + k_ptr, # [blocks, page, Hkv, D] + v_ptr, # [blocks, page, Hkv, D] + rel_ptr, # [total_q, Hq, rel_extent] + out_ptr, # [total_q, Hq, D] + block_table_ptr, # [batch, max_pages] + cache_seqlens, + cu_seqlens_q, + gqa_group_size, + head_dim, + softmax_scale, + stride_q_t, + stride_q_h, + stride_q_d, + stride_k_b, + stride_k_p, + stride_k_h, + stride_k_d, + stride_v_b, + stride_v_p, + stride_v_h, + stride_v_d, + stride_r_t, + stride_r_h, + stride_r_e, + stride_o_t, + stride_o_h, + stride_o_d, + stride_bt_b, + page_size: tl.constexpr, + rel_extent: tl.constexpr, + window_left: tl.constexpr, + BLOCK_Q: tl.constexpr, + BLOCK_K: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_QH: tl.constexpr, +): + # A program owns BLOCK_Q query positions and every query head sharing one + # KV head. This makes the QK/PV operations MFMA-friendly on CDNA. + pid_q = tl.program_id(0) + pid_kh = tl.program_id(1) + pid_b = tl.program_id(2) + + q_start = tl.load(cu_seqlens_q + pid_b) + q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start + q_block = pid_q * BLOCK_Q + if q_block >= q_len: + return + + kv_len = tl.load(cache_seqlens + pid_b) + prefix_len = kv_len - q_len + q_head_start = pid_kh * gqa_group_size + bt_row = block_table_ptr + pid_b * stride_bt_b + + q_block_ptr = tl.make_block_ptr( + base=q_ptr + q_start * stride_q_t + q_head_start * stride_q_h, + shape=(q_len, gqa_group_size, head_dim), + strides=(stride_q_t, stride_q_h, stride_q_d), + offsets=(q_block, 0, 0), + block_shape=(BLOCK_Q, BLOCK_H, BLOCK_D), + order=(2, 1, 0), + ) + q = tl.load(q_block_ptr, boundary_check=(0, 1, 2), padding_option="zero") + q = tl.reshape(q, (BLOCK_QH, BLOCK_D)) + + q_rows = q_block + tl.arange(0, BLOCK_Q) + q_abs = prefix_len + q_rows + q_valid = q_rows < q_len + off_d = tl.arange(0, BLOCK_D) + d_valid = off_d < head_dim + + m_i = tl.full((BLOCK_QH,), float("-inf"), dtype=tl.float32) + l_i = tl.zeros((BLOCK_QH,), dtype=tl.float32) + acc = tl.zeros((BLOCK_QH, BLOCK_D), dtype=tl.float32) + log2e: tl.constexpr = 1.4426950408889634 + + for k_start in range(0, kv_len, BLOCK_K): + k_pos = k_start + tl.arange(0, BLOCK_K) + k_valid = k_pos < kv_len + # Masked lanes still need an in-range page-table load. + safe_pos = tl.minimum(k_pos, tl.maximum(kv_len - 1, 0)) + page = tl.load(bt_row + safe_pos // page_size).to(tl.int64) + page_off = safe_pos % page_size + + k = tl.load( + k_ptr + + page[None, :] * stride_k_b + + page_off[None, :] * stride_k_p + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d, + mask=d_valid[:, None] & k_valid[None, :], + other=0.0, + ) + scores = tl.dot(q, k) * (softmax_scale * log2e) + + dist = q_abs[:, None] - k_pos[None, :] + score_valid = q_valid[:, None] & k_valid[None, :] & (dist >= 0) + if window_left >= 0: + score_valid &= dist <= window_left + + # rel_logits is query- and head-dependent. Expand [Q, K] distance + # indices across the GQA heads, then flatten to match QK's row order. + off_h = tl.arange(0, BLOCK_H) + rel_dist = dist[:, None, :] + rel_valid = ( + q_valid[:, None, None] + & (off_h[None, :, None] < gqa_group_size) + & (rel_dist >= 0) + & (rel_dist < rel_extent) + ) + safe_dist = tl.maximum(0, tl.minimum(rel_dist, rel_extent - 1)) + bias = tl.load( + rel_ptr + + (q_start + q_rows[:, None, None]) * stride_r_t + + (q_head_start + off_h[None, :, None]) * stride_r_h + + safe_dist * stride_r_e, + mask=rel_valid, + other=0.0, + ) + bias = tl.reshape(bias, (BLOCK_QH, BLOCK_K)) + scores += bias.to(tl.float32) * log2e + score_valid = tl.reshape( + score_valid[:, None, :] & (off_h[None, :, None] < gqa_group_size), + (BLOCK_QH, BLOCK_K), + ) + scores = tl.where(score_valid, scores, float("-inf")) + + m_ij = tl.maximum(m_i, tl.max(scores, axis=1)) + has_scores = tl.sum(score_valid.to(tl.int32), axis=1) > 0 + # A sliding-window query can have entire early KV tiles masked. Avoid + # the -inf - -inf NaN in online softmax until its first live tile. + alpha = tl.where(has_scores, tl.exp2(m_i - m_ij), 1.0) + p = tl.where( + score_valid, + tl.exp2(scores - m_ij[:, None]), + 0.0, + ) + l_i = l_i * alpha + tl.sum(p, axis=1) + acc *= alpha[:, None] + + v = tl.load( + v_ptr + + page[:, None] * stride_v_b + + page_off[:, None] * stride_v_p + + pid_kh * stride_v_h + + off_d[None, :] * stride_v_d, + mask=k_valid[:, None] & d_valid[None, :], + other=0.0, + ) + acc += tl.dot(p.to(v.dtype), v) + m_i = m_ij + + acc /= l_i[:, None] + acc = tl.reshape(acc, (BLOCK_Q, BLOCK_H, BLOCK_D)) + out_block_ptr = tl.make_block_ptr( + base=out_ptr + q_start * stride_o_t + q_head_start * stride_o_h, + shape=(q_len, gqa_group_size, head_dim), + strides=(stride_o_t, stride_o_h, stride_o_d), + offsets=(q_block, 0, 0), + block_shape=(BLOCK_Q, BLOCK_H, BLOCK_D), + order=(2, 1, 0), + ) + tl.store( + out_block_ptr, + acc.to(out_ptr.dtype.element_ty), + boundary_check=(0, 1, 2), + ) + + +@torch.no_grad() +def inkling_fa4_rel_attention( + q: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + *, + block_table: torch.Tensor, + cache_seqlens: torch.Tensor, + cu_seqlens_q: torch.Tensor, + max_seqlen_q: int, + softmax_scale: float, + causal: bool, + window_size: tuple[int, int], + rel_extent: int, + rel_logits: torch.Tensor, + num_splits: int = 32, + max_kv_len: int | None = None, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Paged varlen attention with Inkling's relative score modification.""" + del num_splits + if not causal or window_size[1] not in (0, -1): + raise NotImplementedError("Inkling ROCm attention requires causal masking") + if q.ndim != 3 or key_cache.ndim != 4 or value_cache.ndim != 4: + raise ValueError("expected q [T,H,D] and paged K/V [B,P,Hkv,D]") + if rel_logits.shape != (q.shape[0], q.shape[1], rel_extent): + raise ValueError( + f"relative logits have shape {tuple(rel_logits.shape)}, expected " + f"{(q.shape[0], q.shape[1], rel_extent)}" + ) + + num_kv_heads = key_cache.shape[2] + if q.shape[1] % num_kv_heads: + raise ValueError("query heads must be divisible by KV heads") + if out is None: + out = torch.empty_like(q) + gqa_group_size = q.shape[1] // num_kv_heads + batch = cu_seqlens_q.shape[0] - 1 + if max_kv_len is None: + max_kv_len = key_cache.shape[0] * key_cache.shape[1] + if use_gfx950_gluon_decode( + max_query_len=max_seqlen_q, + page_size=key_cache.shape[1], + head_dim=q.shape[2], + ): + # Import only after the architecture guard. The implementation uses + # gfx950-only CDNA4 Gluon layouts and async-copy operations. + from vllm.models.inkling.amd.ops.gluon.rel_mha_decode_gfx950 import ( + gluon_rel_mha_decode_gfx950, + ) + + return gluon_rel_mha_decode_gfx950( + q, + key_cache, + value_cache, + block_table, + cache_seqlens, + max_kv_len, + rel_logits, + cu_seqlens_q, + max_seqlen_q=max_seqlen_q, + window_left=window_size[0], + softmax_scale=softmax_scale, + out=out, + ) + if use_gfx950_gluon_extend( + max_query_len=max_seqlen_q, + max_kv_len=max_kv_len, + page_size=key_cache.shape[1], + head_dim=q.shape[2], + window_left=window_size[0], + ): + from vllm.models.inkling.amd.ops.gluon.rel_mha_extend_gfx950 import ( + gluon_rel_mha_extend_gfx950, + ) + + # The copied kernel retains TokenSpeed's cu_seqlens_kv parameter for + # API compatibility but does not consume it. + return cast( + torch.Tensor, + gluon_rel_mha_extend_gfx950( + q, + cu_seqlens_q, + cu_seqlens_q, + key_cache, + value_cache, + block_table, + cache_seqlens, + window_left=window_size[0], + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_kv_len, + rel_logits=rel_logits, + softmax_scale=softmax_scale, + out=out, + ), + ) + if use_split_kv_decode( + max_query_len=max_seqlen_q, + max_kv_len=max_kv_len, + page_size=key_cache.shape[1], + window_left=window_size[0], + ): + return inkling_rel_attention_split_kv_decode( + q, + key_cache, + value_cache, + block_table=block_table, + cache_seqlens=cache_seqlens, + softmax_scale=softmax_scale, + window_left=window_size[0], + rel_extent=rel_extent, + rel_logits=rel_logits, + max_kv_len=max_kv_len, + out=out, + ) + block_q = 1 if max_seqlen_q == 1 else 4 + grid = (triton.cdiv(max_seqlen_q, block_q), num_kv_heads, batch) + _inkling_rel_attention_kernel[grid]( + q, + key_cache, + value_cache, + rel_logits, + out, + block_table, + cache_seqlens, + cu_seqlens_q, + gqa_group_size, + q.shape[2], + softmax_scale, + q.stride(0), + q.stride(1), + q.stride(2), + key_cache.stride(0), + key_cache.stride(1), + key_cache.stride(2), + key_cache.stride(3), + value_cache.stride(0), + value_cache.stride(1), + value_cache.stride(2), + value_cache.stride(3), + rel_logits.stride(0), + rel_logits.stride(1), + rel_logits.stride(2), + out.stride(0), + out.stride(1), + out.stride(2), + block_table.stride(0), + page_size=key_cache.shape[1], + rel_extent=rel_extent, + window_left=window_size[0], + BLOCK_Q=block_q, + BLOCK_K=64, + num_warps=4, + num_stages=1, + ) + return out diff --git a/vllm/models/inkling/amd/ops/fa4_warmup.py b/vllm/models/inkling/amd/ops/fa4_warmup.py new file mode 100644 index 0000000000..998acdc097 --- /dev/null +++ b/vllm/models/inkling/amd/ops/fa4_warmup.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Compatibility shim for the ROCm relative-attention implementation. + +ROCm uses a Triton kernel which is compiled by vLLM's normal model warmup. The +NVIDIA path registers ahead-of-time CuTeDSL units; importing that provider on +ROCm would pull in CUDA-only tml-fa4 code. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import torch + + +@dataclass(frozen=True) +class InklingFA4WarmupConfig: + num_heads: int + num_kv_heads: int + head_dim: int + rel_extent: int + window_size: tuple[int, int] + is_local: bool + max_kv_len: int + dtype: torch.dtype + kv_dtype: torch.dtype + block_size: int + max_num_reqs: int + max_num_batched_tokens: int + + +def register_fa4_warmup(config: InklingFA4WarmupConfig) -> None: + """Triton compilation is triggered by the ordinary vLLM warmup forward.""" + del config diff --git a/vllm/models/inkling/amd/ops/gluon/__init__.py b/vllm/models/inkling/amd/ops/gluon/__init__.py new file mode 100644 index 0000000000..54494063da --- /dev/null +++ b/vllm/models/inkling/amd/ops/gluon/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""GFX950-only Gluon relative-attention kernels for Inkling.""" diff --git a/vllm/models/inkling/amd/ops/gluon/rel_mha_decode_gfx950.py b/vllm/models/inkling/amd/ops/gluon/rel_mha_decode_gfx950.py new file mode 100644 index 0000000000..987508f91f --- /dev/null +++ b/vllm/models/inkling/amd/ops/gluon/rel_mha_decode_gfx950.py @@ -0,0 +1,1037 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Copyright (c) 2026 LightSeek Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""rel_mha decode Gluon kernel for AMD GFX950.""" + +import math +from typing import NamedTuple + +import torch + +from vllm.models.inkling.amd.ops.gluon.utils import ( + _INV_LN2_VALUE, + InputStrides, + PagedKVStrides, + max, + maximum, +) +from vllm.triton_utils import aggregate, gl, gluon + +cdna4 = gl.amd.cdna4 +async_copy = cdna4.async_copy +cdiv = gl.cdiv +_GFX950_SM_COUNT = 256 + + +# ===-----------------------------------------------------------------------===# +# Kernel Config +# ===-----------------------------------------------------------------------===# + + +@aggregate +class AttentionConfig: + SM_SCALE: gl.constexpr + PAGE_TABLE_STRIDE: gl.constexpr + PAGE_SIZE: gl.constexpr + NUM_KV_SPLITS: gl.constexpr + MAX_SEQLEN_Q: gl.constexpr + NUM_Q_HEADS: gl.constexpr + NUM_KV_HEADS: gl.constexpr + HEAD_DIM: gl.constexpr + BLOCK_M: gl.constexpr + BLOCK_N: gl.constexpr + IS_SLIDING: gl.constexpr + WINDOW_LEFT: gl.constexpr + REL_EXTENT: gl.constexpr + REL_BIAS_QK_SCALE: gl.constexpr + IS_FP8: gl.constexpr + GROUP_SIZE: gl.constexpr + NUM_GROUPS: gl.constexpr + q_strides: InputStrides + rel_strides: InputStrides + k_strides: PagedKVStrides + v_strides: PagedKVStrides + qk_layout: gl.constexpr + pv_layout: gl.constexpr + q_layout: gl.constexpr + k_layout: gl.constexpr + p_layout: gl.constexpr + v_layout: gl.constexpr + load_layout: gl.constexpr + store_layout: gl.constexpr + reduce_layout: gl.constexpr + k_smem_layout: gl.constexpr + v_smem_layout: gl.constexpr + + @gluon.constexpr_function + def __init__( + self, + SM_SCALE, + PAGE_TABLE_STRIDE, + PAGE_SIZE, + NUM_KV_SPLITS, + MAX_SEQLEN_Q, + NUM_Q_HEADS, + NUM_KV_HEADS, + HEAD_DIM, + BLOCK_M, + BLOCK_N, + IS_SLIDING, + WINDOW_LEFT, + REL_EXTENT, + REL_BIAS_QK_SCALE, + IS_FP8, + q_strides, + rel_strides, + k_strides, + v_strides, + ): + assert NUM_Q_HEADS % NUM_KV_HEADS == 0 + assert HEAD_DIM in (64, 128) + assert BLOCK_N == PAGE_SIZE + if IS_SLIDING: + assert WINDOW_LEFT >= 0 + else: + assert WINDOW_LEFT == -1 + + mfma_layout = gl.amd.AMDMFMALayout( + version=4, + instr_shape=[16, 16, 32], + transposed=True, + warps_per_cta=[1, 1], + ) + qk_layout = mfma_layout + pv_layout = mfma_layout + # qk_kw is derived from a 128-bit load / dtype bitwidth. + # pv_kw is empirically tuned. + qk_kw = 16 if IS_FP8 else 8 + pv_kw = 8 if IS_FP8 else 4 + q_layout = gl.DotOperandLayout(0, qk_layout, k_width=qk_kw) + k_layout = gl.DotOperandLayout(1, qk_layout, k_width=qk_kw) + p_layout = gl.DotOperandLayout(0, pv_layout, k_width=pv_kw) + v_layout = gl.DotOperandLayout(1, pv_layout, k_width=pv_kw) + # Elements loaded per lane depend on the input dtype, matching qk_kw. + # load_threads is how many lanes span HEAD_DIM. + load_vec = 16 if IS_FP8 else 8 + load_threads = HEAD_DIM // load_vec + load_layout = gl.BlockedLayout( + [1, load_vec], [64 // load_threads, load_threads], [1, 1], [1, 0] + ) + # Output is always 16-bit, so a 128-bit store has 8 elements. + # store_threads is how many lanes span HEAD_DIM. + store_vec = 8 + store_threads = HEAD_DIM // store_vec + store_layout = gl.BlockedLayout( + [1, store_vec], [64 // store_threads, store_threads], [1, 1], [1, 0] + ) + reduce_layout = gl.BlockedLayout([1, HEAD_DIM // 64], [1, 64], [1, 1], [1, 0]) + # Padding interval is 64 lanes * load_vec elems. + pad_interval = 64 * load_vec + # Empirically tuned. + pad_k = 16 if IS_FP8 else 8 + pad_v = 32 + k_smem_layout = gl.PaddedSharedLayout.with_identity_for( + [[pad_interval, pad_k]], [BLOCK_N, HEAD_DIM], [1, 0] + ) + v_smem_layout = gl.PaddedSharedLayout.with_identity_for( + [[pad_interval, pad_v]], [BLOCK_N, HEAD_DIM], [1, 0] + ) + + self.SM_SCALE = gl.constexpr(SM_SCALE) + self.PAGE_TABLE_STRIDE = gl.constexpr(PAGE_TABLE_STRIDE) + self.PAGE_SIZE = gl.constexpr(PAGE_SIZE) + self.NUM_KV_SPLITS = gl.constexpr(NUM_KV_SPLITS) + self.MAX_SEQLEN_Q = gl.constexpr(MAX_SEQLEN_Q) + self.NUM_Q_HEADS = gl.constexpr(NUM_Q_HEADS) + self.NUM_KV_HEADS = gl.constexpr(NUM_KV_HEADS) + self.HEAD_DIM = gl.constexpr(HEAD_DIM) + self.BLOCK_M = gl.constexpr(BLOCK_M) + self.BLOCK_N = gl.constexpr(BLOCK_N) + self.IS_SLIDING = gl.constexpr(IS_SLIDING) + self.WINDOW_LEFT = gl.constexpr(WINDOW_LEFT) + self.REL_EXTENT = gl.constexpr(REL_EXTENT) + self.REL_BIAS_QK_SCALE = gl.constexpr(REL_BIAS_QK_SCALE) + self.IS_FP8 = gl.constexpr(IS_FP8) + self.GROUP_SIZE = gl.constexpr(NUM_Q_HEADS // NUM_KV_HEADS) + self.NUM_GROUPS = gl.constexpr((self.GROUP_SIZE + BLOCK_M - 1) // BLOCK_M) + self.q_strides = q_strides + self.rel_strides = rel_strides + self.k_strides = k_strides + self.v_strides = v_strides + self.qk_layout = gl.constexpr(qk_layout) + self.pv_layout = gl.constexpr(pv_layout) + self.q_layout = gl.constexpr(q_layout) + self.k_layout = gl.constexpr(k_layout) + self.p_layout = gl.constexpr(p_layout) + self.v_layout = gl.constexpr(v_layout) + self.load_layout = gl.constexpr(load_layout) + self.store_layout = gl.constexpr(store_layout) + self.reduce_layout = gl.constexpr(reduce_layout) + self.k_smem_layout = gl.constexpr(k_smem_layout) + self.v_smem_layout = gl.constexpr(v_smem_layout) + + +# ===-----------------------------------------------------------------------===# +# Kernel Program +# ===-----------------------------------------------------------------------===# + + +@aggregate +class AttentionProgram: + cfg: gl.constexpr + q_ptr: gl.tensor + rel_logits_ptr: gl.tensor + k_cache_ptr: gl.tensor + v_cache_ptr: gl.tensor + page_table_ptr: gl.tensor + cache_seqlens_ptr: gl.tensor + mid_o_ptr: gl.tensor + mid_lse_ptr: gl.tensor + q_index: gl.tensor + batch: gl.tensor + kv_head: gl.tensor + group_start: gl.tensor + split_id: gl.tensor + cache_len: gl.tensor + kv_start: gl.tensor + split_start: gl.tensor + split_end: gl.tensor + + @gluon.constexpr_function + def __init__( + self, + cfg, + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + cache_seqlens_ptr, + mid_o_ptr, + mid_lse_ptr, + q_index, + batch, + kv_head, + group_start, + split_id, + cache_len, + kv_start, + split_start, + split_end, + ): + self.cfg = gl.constexpr(cfg) + self.q_ptr = q_ptr + self.rel_logits_ptr = rel_logits_ptr + self.k_cache_ptr = k_cache_ptr + self.v_cache_ptr = v_cache_ptr + self.page_table_ptr = page_table_ptr + self.cache_seqlens_ptr = cache_seqlens_ptr + self.mid_o_ptr = mid_o_ptr + self.mid_lse_ptr = mid_lse_ptr + self.q_index = q_index + self.batch = batch + self.kv_head = kv_head + self.group_start = group_start + self.split_id = split_id + self.cache_len = cache_len + self.kv_start = kv_start + self.split_start = split_start + self.split_end = split_end + + @gluon.jit + def create( + cfg, + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + cache_seqlens_ptr, + mid_o_ptr, + mid_lse_ptr, + ): + # Gluon treats ``cfg`` as the constexpr factory argument, while mypy + # models the first parameter of this aggregate method as ``self``. + q_index = gl.program_id(0) + batch = q_index // cfg.MAX_SEQLEN_Q # type: ignore[attr-defined] + q_pos = q_index - batch * cfg.MAX_SEQLEN_Q # type: ignore[attr-defined] + head_block = gl.program_id(1) + kv_head = head_block // cfg.NUM_GROUPS # type: ignore[attr-defined] + group_block = head_block - kv_head * cfg.NUM_GROUPS # type: ignore[attr-defined] + group_start = group_block * cfg.BLOCK_M # type: ignore[attr-defined] + split_id = gl.program_id(2) + cache_len = gl.load(cache_seqlens_ptr + batch) + cache_len = cache_len - ( + cfg.MAX_SEQLEN_Q - 1 - q_pos # type: ignore[attr-defined] + ) + cache_len = maximum(cache_len, 0) + if cfg.IS_SLIDING: # type: ignore[attr-defined] + # WINDOW_LEFT is defined as exclusive (keys strictly to the left, not + # counting the current token), so the window is WINDOW_LEFT + 1 keys + # once the current token is included. E.g. with + # WINDOW_LEFT = 127 and cache_len = 500 (current token at index 499), + # kv_start = 500 - (127 + 1) = 372 keeps keys [372, 499] = 128 keys. + # Without the + 1, kv_start = 373 would drop the leftmost key. + kv_start = cache_len - min( + cache_len, + cfg.WINDOW_LEFT + 1, # type: ignore[attr-defined] + ) + else: + kv_start = cache_len - cache_len + first_page = kv_start // cfg.PAGE_SIZE # type: ignore[attr-defined] + end_page = cdiv(cache_len, cfg.PAGE_SIZE) # type: ignore[attr-defined] + num_pages = end_page - first_page + pages_per_split = cdiv( + num_pages, + cfg.NUM_KV_SPLITS, # type: ignore[attr-defined] + ) + split_start_page = first_page + split_id * pages_per_split + split_end_page = min(split_start_page + pages_per_split, end_page) + split_start = ( + split_start_page * cfg.PAGE_SIZE # type: ignore[attr-defined] + ) + split_end = min( + split_end_page * cfg.PAGE_SIZE, # type: ignore[attr-defined] + cache_len, + ) + return AttentionProgram( + gl.constexpr(cfg), + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + cache_seqlens_ptr, + mid_o_ptr, + mid_lse_ptr, + q_index, + batch, + kv_head, + group_start, + split_id, + cache_len, + kv_start, + split_start, + split_end, + ) + + @gluon.jit + def load_q(self): + cfg = self.cfg + offs_m = gl.arange(0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.q_layout)) + offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.q_layout)) + q_heads = self.kv_head * cfg.GROUP_SIZE + self.group_start + offs_m + valid = (self.group_start + offs_m) < cfg.GROUP_SIZE + offsets = cfg.q_strides.offsets(self.q_index, q_heads[:, None], offs_d[None, :]) + return cdna4.buffer_load(self.q_ptr, offsets, mask=valid[:, None], other=0.0) + + @gluon.jit + def init_state(self): + cfg = self.cfg + m_i = gl.full( + [cfg.BLOCK_M], + value=-float("inf"), + dtype=gl.float32, + layout=gl.SliceLayout(1, cfg.pv_layout), + ) + l_i = gl.full( + [cfg.BLOCK_M], + value=0.0, + dtype=gl.float32, + layout=gl.SliceLayout(1, cfg.pv_layout), + ) + acc = gl.zeros( + [cfg.BLOCK_M, cfg.HEAD_DIM], dtype=gl.float32, layout=cfg.pv_layout + ) + return m_i, l_i, acc + + @gluon.jit + def load_page(self, start_n): + cfg = self.cfg + page_index = start_n // cfg.PAGE_SIZE + valid = start_n < self.split_end + return gl.load( + self.page_table_ptr + self.batch * cfg.PAGE_TABLE_STRIDE + page_index, + mask=valid, + other=0, + ) + + @gluon.jit + def issue_load_k(self, physical_page, k_smem): + cfg = self.cfg + offs_n = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(1, cfg.load_layout)) + offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.load_layout)) + offsets = cfg.k_strides.offsets( + physical_page, + offs_n[:, None], + self.kv_head, + offs_d[None, :], + ) + # can't use buffer_load: paged KV offsets may exceed its 32-bit range. + async_copy.global_load_to_shared(k_smem, self.k_cache_ptr + offsets) + async_copy.commit_group() + + @gluon.jit + def issue_load_v(self, physical_page, v_smem): + cfg = self.cfg + offs_n = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(1, cfg.load_layout)) + offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.load_layout)) + offsets = cfg.v_strides.offsets( + physical_page, + offs_n[:, None], + self.kv_head, + offs_d[None, :], + ) + # can't use buffer_load: paged KV offsets may exceed its 32-bit range. + async_copy.global_load_to_shared(v_smem, self.v_cache_ptr + offsets) + async_copy.commit_group() + + @gluon.jit + def shared_load_k(self, k_smem): + return k_smem.permute([1, 0]).load(self.cfg.k_layout) + + @gluon.jit + def shared_load_v(self, v_smem): + return v_smem.load(self.cfg.v_layout) + + @gluon.jit + def compute_qk(self, q, k): + cfg = self.cfg + qk = gl.zeros( + [cfg.BLOCK_M, cfg.BLOCK_N], dtype=gl.float32, layout=cfg.qk_layout + ) + return cdna4.mfma(q, k, qk) + + @gluon.jit + def apply_rel_bias(self, qk, start_n): + cfg = self.cfg + offs_m = gl.arange(0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.qk_layout)) + offs_n = start_n + gl.arange( + 0, cfg.BLOCK_N, layout=gl.SliceLayout(0, cfg.qk_layout) + ) + q_heads = self.kv_head * cfg.GROUP_SIZE + self.group_start + offs_m + rel_dist = (self.cache_len - 1) - offs_n + rel_valid = (rel_dist >= 0) & (rel_dist < cfg.REL_EXTENT) + rel_idx = gl.where(rel_dist >= 0, rel_dist, 0) + rel_idx = gl.where(rel_idx < cfg.REL_EXTENT, rel_idx, cfg.REL_EXTENT - 1) + offsets = cfg.rel_strides.offsets( + self.q_index, q_heads[:, None], rel_idx[None, :] + ) + head_valid = (self.group_start + offs_m) < cfg.GROUP_SIZE + rel_bias = cdna4.buffer_load( + self.rel_logits_ptr, + offsets, + mask=head_valid[:, None] & rel_valid[None, :], + other=0.0, + ).to(gl.float32) + return qk + rel_bias * cfg.REL_BIAS_QK_SCALE + + @gluon.jit + def apply_kv_mask(self, qk, start_n): + cfg = self.cfg + offs_n = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(0, cfg.qk_layout)) + tokens = start_n + offs_n[None, :] + mask = (tokens >= self.kv_start) & (tokens < self.split_end) + return gl.where(mask, qk, -float("inf")) + + @gluon.jit + def softmax(self, qk, m_i, l_i, acc): + cfg = self.cfg + row_max = max(qk, axis=1) + row_max = gl.convert_layout(row_max, gl.SliceLayout(1, cfg.pv_layout)) + m_new = maximum(m_i, row_max) + m_new_scaled = m_new * cfg.SM_SCALE + qk_shifted = qk * cfg.SM_SCALE - m_new_scaled[:, None] + p = gl.exp2(qk_shifted) + m_diff = m_i * cfg.SM_SCALE - m_new_scaled + alpha = gl.exp2(m_diff) + l_ij = gl.sum(p, axis=1) + l_i = l_i * alpha + l_ij + acc = acc * alpha[:, None] + p = p.to(self.q_ptr.dtype.element_ty) + p = gl.convert_layout(p, cfg.p_layout) + return p, m_new, l_i, acc + + @gluon.jit + def compute_pv(self, p, v, acc): + return cdna4.mfma(p, v, acc) + + @gluon.jit + def store_split(self, acc, l_i, m_i): + cfg = self.cfg + offs_m = gl.arange(0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.store_layout)) + offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.store_layout)) + q_heads = self.kv_head * cfg.GROUP_SIZE + self.group_start + offs_m + valid = ((self.group_start + offs_m) < cfg.GROUP_SIZE) & ( + self.split_start < self.split_end + ) + acc = gl.convert_layout(acc, cfg.store_layout) + l_i = gl.convert_layout(l_i, gl.SliceLayout(1, cfg.store_layout)) + m_i = gl.convert_layout(m_i, gl.SliceLayout(1, cfg.store_layout)) + recip_l_i = 1.0 / l_i + part_o = acc * recip_l_i[:, None] + part_lse = m_i * cfg.SM_SCALE + gl.log2(l_i) + mid_o_offsets = ( + (self.q_index * cfg.NUM_Q_HEADS + q_heads[:, None]) * cfg.NUM_KV_SPLITS + + self.split_id + ) * cfg.HEAD_DIM + offs_d[None, :] + mid_lse_offsets = ( + self.q_index * cfg.NUM_Q_HEADS + q_heads + ) * cfg.NUM_KV_SPLITS + self.split_id + cdna4.buffer_store(part_o, self.mid_o_ptr, mid_o_offsets, mask=valid[:, None]) + cdna4.buffer_store(part_lse, self.mid_lse_ptr, mid_lse_offsets, mask=valid) + + @gluon.jit + def store_output(self, acc, l_i): + cfg = self.cfg + offs_m = gl.arange(0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.store_layout)) + offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.store_layout)) + q_heads = self.kv_head * cfg.GROUP_SIZE + self.group_start + offs_m + valid = (self.group_start + offs_m) < cfg.GROUP_SIZE + acc = gl.convert_layout(acc, cfg.store_layout) + l_i = gl.convert_layout(l_i, gl.SliceLayout(1, cfg.store_layout)) + output = acc * (1.0 / l_i)[:, None] + output = output.to(self.mid_o_ptr.dtype.element_ty) + offsets = (self.q_index * cfg.NUM_Q_HEADS + q_heads[:, None]) * cfg.HEAD_DIM + offsets += offs_d[None, :] + cdna4.buffer_store(output, self.mid_o_ptr, offsets, mask=valid[:, None]) + + +# ===-----------------------------------------------------------------------===# +# Entry Point +# ===-----------------------------------------------------------------------===# + + +@gluon.jit +def _rel_mha_decode_fp16( + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + cache_seqlens_ptr, + mid_o_ptr, + mid_lse_ptr, + Q_STRIDE_B: gl.constexpr, + Q_STRIDE_H: gl.constexpr, + Q_STRIDE_D: gl.constexpr, + K_STRIDE_B: gl.constexpr, + K_STRIDE_P: gl.constexpr, + K_STRIDE_H: gl.constexpr, + K_STRIDE_D: gl.constexpr, + V_STRIDE_B: gl.constexpr, + V_STRIDE_P: gl.constexpr, + V_STRIDE_H: gl.constexpr, + V_STRIDE_D: gl.constexpr, + SM_SCALE: gl.constexpr, + PAGE_TABLE_STRIDE: gl.constexpr, + PAGE_SIZE: gl.constexpr, + NUM_KV_SPLITS: gl.constexpr, + MAX_SEQLEN_Q: gl.constexpr, + NUM_Q_HEADS: gl.constexpr, + NUM_KV_HEADS: gl.constexpr, + HEAD_DIM: gl.constexpr, + BLOCK_M: gl.constexpr, + BLOCK_N: gl.constexpr, + IS_SLIDING: gl.constexpr, + WINDOW_LEFT: gl.constexpr, + REL_STRIDE_T: gl.constexpr, + REL_STRIDE_H: gl.constexpr, + REL_STRIDE_E: gl.constexpr, + REL_EXTENT: gl.constexpr, + REL_BIAS_QK_SCALE: gl.constexpr, + IS_FP8: gl.constexpr, +): + cfg = AttentionConfig( + SM_SCALE, + PAGE_TABLE_STRIDE, + PAGE_SIZE, + NUM_KV_SPLITS, + MAX_SEQLEN_Q, + NUM_Q_HEADS, + NUM_KV_HEADS, + HEAD_DIM, + BLOCK_M, + BLOCK_N, + IS_SLIDING, + WINDOW_LEFT, + REL_EXTENT, + REL_BIAS_QK_SCALE, + IS_FP8, + InputStrides(Q_STRIDE_B, Q_STRIDE_H, Q_STRIDE_D), + InputStrides(REL_STRIDE_T, REL_STRIDE_H, REL_STRIDE_E), + PagedKVStrides(K_STRIDE_B, K_STRIDE_P, K_STRIDE_H, K_STRIDE_D), + PagedKVStrides(V_STRIDE_B, V_STRIDE_P, V_STRIDE_H, V_STRIDE_D), + ) + program = AttentionProgram.create( + cfg, + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + cache_seqlens_ptr, + mid_o_ptr, + mid_lse_ptr, + ) + k_smem = gl.allocate_shared_memory( + k_cache_ptr.dtype.element_ty, [cfg.BLOCK_N, cfg.HEAD_DIM], cfg.k_smem_layout + ) + v_smem = gl.allocate_shared_memory( + v_cache_ptr.dtype.element_ty, [cfg.BLOCK_N, cfg.HEAD_DIM], cfg.v_smem_layout + ) + + q = program.load_q() + m_i, l_i, acc = program.init_state() + + physical_page = program.load_page(program.split_start) + + for start_n in range(program.split_start, program.split_end, cfg.BLOCK_N): + program.issue_load_k(physical_page, k_smem) + program.issue_load_v(physical_page, v_smem) + physical_page = program.load_page(start_n + cfg.BLOCK_N) + + async_copy.wait_group(1) + k = program.shared_load_k(k_smem) + qk = program.compute_qk(q, k) + qk = program.apply_rel_bias(qk, start_n) + qk = program.apply_kv_mask(qk, start_n) + p, m_i, l_i, acc = program.softmax(qk, m_i, l_i, acc) + + async_copy.wait_group(0) + v = program.shared_load_v(v_smem) + acc = program.compute_pv(p, v, acc) + + program.store_split(acc, l_i, m_i) + + +@gluon.jit +def _rel_mha_decode_sliding_fp16( + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + cache_seqlens_ptr, + out_ptr, + Q_STRIDE_B: gl.constexpr, + Q_STRIDE_H: gl.constexpr, + Q_STRIDE_D: gl.constexpr, + K_STRIDE_B: gl.constexpr, + K_STRIDE_P: gl.constexpr, + K_STRIDE_H: gl.constexpr, + K_STRIDE_D: gl.constexpr, + V_STRIDE_B: gl.constexpr, + V_STRIDE_P: gl.constexpr, + V_STRIDE_H: gl.constexpr, + V_STRIDE_D: gl.constexpr, + SM_SCALE: gl.constexpr, + PAGE_TABLE_STRIDE: gl.constexpr, + PAGE_SIZE: gl.constexpr, + MAX_SEQLEN_Q: gl.constexpr, + NUM_Q_HEADS: gl.constexpr, + NUM_KV_HEADS: gl.constexpr, + HEAD_DIM: gl.constexpr, + BLOCK_M: gl.constexpr, + BLOCK_N: gl.constexpr, + IS_SLIDING: gl.constexpr, + WINDOW_LEFT: gl.constexpr, + REL_STRIDE_T: gl.constexpr, + REL_STRIDE_H: gl.constexpr, + REL_STRIDE_E: gl.constexpr, + REL_EXTENT: gl.constexpr, + REL_BIAS_QK_SCALE: gl.constexpr, + IS_FP8: gl.constexpr, +): + cfg = AttentionConfig( + SM_SCALE, + PAGE_TABLE_STRIDE, + PAGE_SIZE, + 1, + MAX_SEQLEN_Q, + NUM_Q_HEADS, + NUM_KV_HEADS, + HEAD_DIM, + BLOCK_M, + BLOCK_N, + IS_SLIDING, + WINDOW_LEFT, + REL_EXTENT, + REL_BIAS_QK_SCALE, + IS_FP8, + InputStrides(Q_STRIDE_B, Q_STRIDE_H, Q_STRIDE_D), + InputStrides(REL_STRIDE_T, REL_STRIDE_H, REL_STRIDE_E), + PagedKVStrides(K_STRIDE_B, K_STRIDE_P, K_STRIDE_H, K_STRIDE_D), + PagedKVStrides(V_STRIDE_B, V_STRIDE_P, V_STRIDE_H, V_STRIDE_D), + ) + program = AttentionProgram.create( + cfg, + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + cache_seqlens_ptr, + out_ptr, + out_ptr, + ) + k_smem = gl.allocate_shared_memory( + k_cache_ptr.dtype.element_ty, [cfg.BLOCK_N, cfg.HEAD_DIM], cfg.k_smem_layout + ) + v_smem = gl.allocate_shared_memory( + v_cache_ptr.dtype.element_ty, [cfg.BLOCK_N, cfg.HEAD_DIM], cfg.v_smem_layout + ) + + q = program.load_q() + m_i, l_i, acc = program.init_state() + + for start_n in range(program.split_start, program.split_end, cfg.BLOCK_N): + physical_page = program.load_page(start_n) + program.issue_load_k(physical_page, k_smem) + program.issue_load_v(physical_page, v_smem) + async_copy.wait_group(1) + k = program.shared_load_k(k_smem) + qk = program.compute_qk(q, k) + qk = program.apply_rel_bias(qk, start_n) + qk = program.apply_kv_mask(qk, start_n) + p, m_i, l_i, acc = program.softmax(qk, m_i, l_i, acc) + + async_copy.wait_group(0) + v = program.shared_load_v(v_smem) + acc = program.compute_pv(p, v, acc) + + program.store_output(acc, l_i) + + +@gluon.jit +def _rel_mha_decode_reduce_fp16( + mid_o_ptr, + mid_lse_ptr, + out_ptr, + cache_seqlens_ptr, + SM_SCALE: gl.constexpr, + PAGE_TABLE_STRIDE: gl.constexpr, + NUM_KV_SPLITS: gl.constexpr, + MAX_SEQLEN_Q: gl.constexpr, + PAGE_SIZE: gl.constexpr, + NUM_Q_HEADS: gl.constexpr, + NUM_KV_HEADS: gl.constexpr, + HEAD_DIM: gl.constexpr, + BLOCK_M: gl.constexpr, + BLOCK_N: gl.constexpr, + IS_SLIDING: gl.constexpr, + WINDOW_LEFT: gl.constexpr, + IS_FP8: gl.constexpr, +): + cfg = AttentionConfig( + SM_SCALE, + PAGE_TABLE_STRIDE, + PAGE_SIZE, + NUM_KV_SPLITS, + MAX_SEQLEN_Q, + NUM_Q_HEADS, + NUM_KV_HEADS, + HEAD_DIM, + BLOCK_M, + BLOCK_N, + IS_SLIDING, + WINDOW_LEFT, + 1, # REL_EXTENT + 1.0, # REL_BIAS_QK_SCALE + IS_FP8, + InputStrides(1, 1, 1), + InputStrides(1, 1, 1), + PagedKVStrides(1, 1, 1, 1), + PagedKVStrides(1, 1, 1, 1), + ) + q_index = gl.program_id(0) + batch = q_index // MAX_SEQLEN_Q + q_pos = q_index - batch * MAX_SEQLEN_Q + q_head = gl.program_id(1) + cache_len = gl.load(cache_seqlens_ptr + batch) + cache_len = cache_len - (MAX_SEQLEN_Q - 1 - q_pos) + cache_len = maximum(cache_len, 0) + if cfg.IS_SLIDING: + kv_start = cache_len - min(cache_len, cfg.WINDOW_LEFT + 1) + else: + kv_start = cache_len - cache_len + first_page = kv_start // cfg.PAGE_SIZE + end_page = cdiv(cache_len, cfg.PAGE_SIZE) + num_pages = end_page - first_page + pages_per_split = cdiv(num_pages, cfg.NUM_KV_SPLITS) + + # SPLIT_TILE pads NUM_KV_SPLITS up to a power of 2. + SPLIT_TILE: gl.constexpr = 1 << (NUM_KV_SPLITS - 1).bit_length() + offs_s = gl.arange(0, SPLIT_TILE, layout=gl.SliceLayout(1, cfg.reduce_layout)) + offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.reduce_layout)) + # split_valid masks out empty splits and the power-of-2 padding tail. + split_start_page = first_page + offs_s * pages_per_split + split_end_page_raw = split_start_page + pages_per_split + split_end_page = gl.where( + split_end_page_raw < end_page, split_end_page_raw, end_page + ) + split_start_tok = split_start_page * cfg.PAGE_SIZE + split_end_raw = split_end_page * cfg.PAGE_SIZE + split_end_tok = gl.where(split_end_raw < cache_len, split_end_raw, cache_len) + split_valid = (split_start_tok < split_end_tok) & (offs_s < cfg.NUM_KV_SPLITS) + # Load every split's partial output and lse. + base = (q_index * cfg.NUM_Q_HEADS + q_head) * cfg.NUM_KV_SPLITS + offs_s + part_lse = gl.load(mid_lse_ptr + base, mask=split_valid, other=-float("inf")) + o_off = base[:, None] * cfg.HEAD_DIM + offs_d[None, :] + part_o = cdna4.buffer_load(mid_o_ptr, o_off, mask=split_valid[:, None], other=0.0) + + # Global softmax max over all splits. + m_i = max(part_lse, axis=0) + # Weighted sum of the split partials, normalized by the total softmax mass. + beta = gl.exp2(part_lse - m_i) + l_i = gl.sum(beta, axis=0) + acc = gl.sum(part_o * beta[:, None], axis=0) + + out_base = (q_index * cfg.NUM_Q_HEADS + q_head) * cfg.HEAD_DIM + output = acc * (1.0 / l_i) + output = output.to(out_ptr.dtype.element_ty) + cdna4.buffer_store(output, out_ptr, out_base + offs_d) + + +def _select_num_kv_splits( + *, + batch: int, + num_kv_heads: int, + num_groups: int, + num_pages: int, + sm_count: int, +) -> int: + """Pick num_kv_splits to balance occupancy against reduce overhead. + + The launch grid is (batch * num_kv_heads * num_groups) * num_kv_splits + work-groups. Too few splits under-fill the machine at low batch; too many + leave each split with a handful of pages, so the reduce kernel dominates. + + Return the smaller of two candidate counts: splits_for_occupancy (enough to + fill ~wave_target waves of CUs) and splits_for_pages (~min_pages_per_split + pages per split), with the pages candidate clamped to [min_page_splits, + max_page_splits] so a short context still splits without launching empty work + and a long one does not over-split where reduce cost outgrows the decode win. + """ + wave_target = 2 + min_pages_per_split = 2 + min_page_splits = 8 + max_page_splits = 32 + + base_ctas = batch * num_kv_heads * num_groups + target_ctas = sm_count * wave_target + splits_for_occupancy = (target_ctas + base_ctas - 1) // base_ctas + + splits_for_pages = num_pages // min_pages_per_split + min_page_splits = min(min_page_splits, num_pages) + if splits_for_pages < min_page_splits: + splits_for_pages = min_page_splits + if splits_for_pages > max_page_splits: + splits_for_pages = max_page_splits + return min(splits_for_occupancy, splits_for_pages) + + +class LaunchConfig(NamedTuple): + num_q_heads: int + num_kv_heads: int + num_groups: int + head_dim: int + page_size: int + num_kv_splits: int + block_m: int + block_n: int + sm_scale: float + rel_bias_qk_scale: float + is_sliding: bool + window_left: int + + +def get_config( + *, + q: torch.Tensor, + k_cache: torch.Tensor, + max_seqlen_k: int, + window_left: int, + softmax_scale: float | None, +) -> LaunchConfig: + head_dim = q.shape[2] + page_size = k_cache.shape[1] + block_m = 16 + block_n = page_size + group_size = q.shape[1] // k_cache.shape[2] + num_groups = math.ceil(group_size / block_m) + is_sliding = window_left >= 0 + window_left = window_left if is_sliding else -1 + if softmax_scale is None: + softmax_scale = 1.0 / math.sqrt(head_dim) + sm_scale = softmax_scale + effective_seqlen_k = ( + min(max_seqlen_k, window_left + 1) if is_sliding else max_seqlen_k + ) + num_pages = (effective_seqlen_k + page_size - 1) // page_size + if is_sliding: + num_kv_splits = 4 + else: + num_kv_splits = _select_num_kv_splits( + batch=q.shape[0], + num_kv_heads=k_cache.shape[2], + num_groups=num_groups, + num_pages=num_pages, + sm_count=_GFX950_SM_COUNT, + ) + return LaunchConfig( + num_q_heads=q.shape[1], + num_kv_heads=k_cache.shape[2], + num_groups=num_groups, + head_dim=head_dim, + page_size=page_size, + num_kv_splits=num_kv_splits, + block_m=block_m, + block_n=block_n, + sm_scale=sm_scale * _INV_LN2_VALUE, + rel_bias_qk_scale=1.0 / softmax_scale, + is_sliding=is_sliding, + window_left=window_left, + ) + + +def gluon_rel_mha_decode_gfx950( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + page_table: torch.Tensor, + cache_seqlens: torch.Tensor, + max_seqlen_k: int, + rel_logits: torch.Tensor, + cu_seqlens_q: torch.Tensor, + max_seqlen_q: int = 1, + window_left: int = -1, + softmax_scale: float | None = None, + q_scale: torch.Tensor | None = None, + k_scale: torch.Tensor | None = None, + v_scale: torch.Tensor | None = None, + out: torch.Tensor | None = None, +) -> torch.Tensor: + total_q = q.shape[0] + + config = get_config( + q=q, + k_cache=k_cache, + max_seqlen_k=max_seqlen_k, + window_left=window_left, + softmax_scale=softmax_scale, + ) + + is_fp8 = q.dtype in (torch.float8_e4m3fn, torch.float8_e5m2) + out_dtype = torch.bfloat16 if is_fp8 else q.dtype + if k_cache.shape[1] not in (64, 128, 256): + raise ValueError( + "gfx950 Gluon relative-attention decode requires page size " + f"64, 128, or 256; got {k_cache.shape[1]}" + ) + output = ( + torch.empty(q.shape, device=q.device, dtype=out_dtype) if out is None else out + ) + + # Always use split-k for full attention and for the small-batch sliding + # decode path. Sliding uses a fixed 8 splits, one page per split for the + # TP-4 local-attention shape with 512-token window and 64-token pages. + mid_o = torch.empty( + (total_q, config.num_q_heads, config.num_kv_splits, config.head_dim), + device=q.device, + dtype=torch.float32, + ) + mid_lse = torch.empty( + (total_q, config.num_q_heads, config.num_kv_splits), + device=q.device, + dtype=torch.float32, + ) + + grid = ( + total_q, + config.num_kv_heads * config.num_groups, + config.num_kv_splits, + ) + _rel_mha_decode_fp16[grid]( + q, + rel_logits, + k_cache, + v_cache, + page_table, + cache_seqlens, + mid_o, + mid_lse, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride(3), + config.sm_scale, + page_table.stride(0), + config.page_size, + config.num_kv_splits, + max_seqlen_q, + config.num_q_heads, + config.num_kv_heads, + config.head_dim, + config.block_m, + config.block_n, + config.is_sliding, + config.window_left, + rel_logits.stride(0), + rel_logits.stride(1), + rel_logits.stride(2), + rel_logits.shape[2], + config.rel_bias_qk_scale, + is_fp8, + num_warps=1, + ) + + reduce_grid = (total_q, config.num_q_heads) + _rel_mha_decode_reduce_fp16[reduce_grid]( + mid_o, + mid_lse, + output, + cache_seqlens, + config.sm_scale, + page_table.stride(0), + config.num_kv_splits, + max_seqlen_q, + config.page_size, + config.num_q_heads, + config.num_kv_heads, + config.head_dim, + config.block_m, + config.block_n, + config.is_sliding, + config.window_left, + is_fp8, + num_warps=1, + ) + return output diff --git a/vllm/models/inkling/amd/ops/gluon/rel_mha_extend_gfx950.py b/vllm/models/inkling/amd/ops/gluon/rel_mha_extend_gfx950.py new file mode 100644 index 0000000000..224524e76b --- /dev/null +++ b/vllm/models/inkling/amd/ops/gluon/rel_mha_extend_gfx950.py @@ -0,0 +1,760 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Copyright (c) 2026 LightSeek Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""rel_mha extend Gluon kernel for AMD GFX950. + +This handles ragged, multi-token queries against a paged KV cache. The query +axis is tiled into the MFMA ``M`` dimension (prefill-style): ``BLOCK_M`` query +rows of a single q-head share each paged KV tile, so every KV tile is loaded +once and reused across all rows in the tile. The grid is +``(blocks_per_req, batch, n_heads)`` -- all host-known sizes -- and each program +self-locates its request from ``cu_seqlens_q`` / ``cache_seqlens`` in-kernel, so +the launch stays CUDA-graph static with no device->host sync. + +Visibility per query row depends on ``is_causal`` and the optional sliding +window: + +* ``is_causal=False``: every query token attends the full visible cache, i.e. + ``visible_kv = cache_seqlens[batch]``. +* ``is_causal=True``: query tokens are a causal suffix, so the ``i``-th query + token of a request (0-indexed) attends ``prefix + i + 1`` tokens, where + ``prefix = cache_seqlens[batch] - query_len[batch]``. + +Causal masking only touches the few KV tiles that reach the diagonal; the long +prefix is a mask-free fast path. Sinks and sliding windows (causal or not) are +applied per tile. This is the sole Gluon extend implementation. +""" + +import math + +import torch + +from vllm.models.inkling.amd.ops.gluon.utils import ( + _INV_LN2_VALUE, + _LN2, + InputStrides, + PagedKVStrides, + max, + maximum, +) +from vllm.triton_utils import aggregate, gl, gluon + +cdna4 = gl.amd.cdna4 +async_copy = cdna4.async_copy + + +# ===-----------------------------------------------------------------------===# +# Query-batched path +# +# This path tiles BLOCK_M query rows (of a single q-head) into the MFMA M +# dimension -- like the prefill kernel -- so each paged KV tile is loaded once +# and reused across all BLOCK_M rows. KV still comes from the paged cache +# (decode-style async page loads). Causal masking only touches the few KV tiles +# that reach the diagonal; the long prefix is a mask-free fast path. +# ===-----------------------------------------------------------------------===# + +_EXTEND_SHORT_Q_BLOCK_M = 64 +_EXTEND_SHORT_Q_NUM_WARPS = 2 +_EXTEND_LONG_Q_BLOCK_M = 128 +_EXTEND_LONG_Q_NUM_WARPS = 4 + + +def _select_extend_tile(max_seqlen_q: int, page_size: int) -> tuple[int, int, int]: + """Return (BLOCK_M, BLOCK_N, NUM_WARPS) for the given max query length. + + Queries that fit in a single short tile use it (least padding, most + occupancy); longer ones use the tall tile that covers more rows per + shared-KV pass. + """ + if max_seqlen_q <= _EXTEND_SHORT_Q_BLOCK_M: + return _EXTEND_SHORT_Q_BLOCK_M, page_size, _EXTEND_SHORT_Q_NUM_WARPS + return _EXTEND_LONG_Q_BLOCK_M, page_size, _EXTEND_LONG_Q_NUM_WARPS + + +@aggregate +class ExtendConfig: + N_HEADS: gl.constexpr + N_KV_HEADS: gl.constexpr + GROUP_SIZE: gl.constexpr + HEAD_DIM: gl.constexpr + SM_SCALE: gl.constexpr + BLOCK_M: gl.constexpr + BLOCK_N: gl.constexpr + NUM_WARPS: gl.constexpr + PAGE_SIZE: gl.constexpr + PAGE_TABLE_STRIDE: gl.constexpr + IS_CAUSAL: gl.constexpr + HAS_LSE: gl.constexpr + WINDOW_LEFT: gl.constexpr + REL_EXTENT: gl.constexpr + REL_BIAS_QK_SCALE: gl.constexpr + IS_FP8: gl.constexpr + q_strides: InputStrides + rel_strides: InputStrides + k_strides: PagedKVStrides + v_strides: PagedKVStrides + qk_layout: gl.constexpr + pv_layout: gl.constexpr + q_layout: gl.constexpr + k_layout: gl.constexpr + p_layout: gl.constexpr + v_layout: gl.constexpr + load_layout: gl.constexpr + store_layout: gl.constexpr + k_smem_layout: gl.constexpr + v_smem_layout: gl.constexpr + + @gluon.constexpr_function + def __init__( + self, + N_HEADS, + N_KV_HEADS, + HEAD_DIM, + SM_SCALE, + BLOCK_M, + BLOCK_N, + NUM_WARPS, + PAGE_SIZE, + PAGE_TABLE_STRIDE, + IS_CAUSAL, + HAS_LSE, + WINDOW_LEFT, + REL_EXTENT, + REL_BIAS_QK_SCALE, + IS_FP8, + q_strides, + rel_strides, + k_strides, + v_strides, + ): + assert HEAD_DIM in (64, 128) + assert BLOCK_N == PAGE_SIZE + + instr_shape = [32, 32, 16] + mfma_layout = gl.amd.AMDMFMALayout( + version=4, + instr_shape=instr_shape, + transposed=True, + warps_per_cta=[NUM_WARPS, 1], + ) + qk_layout = mfma_layout + pv_layout = mfma_layout + # Elements loaded per lane depend on the input dtype, matching qk_kw. + # load_threads is how many lanes span HEAD_DIM. + load_vec = 16 if IS_FP8 else 8 + load_threads = HEAD_DIM // load_vec + load_layout = gl.BlockedLayout( + [1, load_vec], [64 // load_threads, load_threads], [NUM_WARPS, 1], [1, 0] + ) + # Output is always 16-bit, so a 128-bit store has 8 elements. + # store_threads is how many lanes span HEAD_DIM. + store_vec = 8 + store_threads = HEAD_DIM // store_vec + store_layout = gl.BlockedLayout( + [1, store_vec], [64 // store_threads, store_threads], [NUM_WARPS, 1], [1, 0] + ) + # Padding interval is 64 lanes * load_vec elems. + pad_interval = 64 * load_vec + # Empirically tuned. + pad_k = 16 if IS_FP8 else 8 + pad_v = 16 if IS_FP8 else 32 + k_smem_layout = gl.PaddedSharedLayout.with_identity_for( + [[pad_interval, pad_k]], [BLOCK_N, HEAD_DIM], [1, 0] + ) + v_smem_layout = gl.PaddedSharedLayout.with_identity_for( + [[pad_interval, pad_v]], [BLOCK_N, HEAD_DIM], [1, 0] + ) + + self.N_HEADS = gl.constexpr(N_HEADS) + self.N_KV_HEADS = gl.constexpr(N_KV_HEADS) + self.GROUP_SIZE = gl.constexpr(N_HEADS // N_KV_HEADS) + self.HEAD_DIM = gl.constexpr(HEAD_DIM) + self.SM_SCALE = gl.constexpr(SM_SCALE) + self.BLOCK_M = gl.constexpr(BLOCK_M) + self.BLOCK_N = gl.constexpr(BLOCK_N) + self.NUM_WARPS = gl.constexpr(NUM_WARPS) + self.PAGE_SIZE = gl.constexpr(PAGE_SIZE) + self.PAGE_TABLE_STRIDE = gl.constexpr(PAGE_TABLE_STRIDE) + self.IS_CAUSAL = gl.constexpr(IS_CAUSAL) + self.HAS_LSE = gl.constexpr(HAS_LSE) + self.WINDOW_LEFT = gl.constexpr(WINDOW_LEFT) + self.REL_EXTENT = gl.constexpr(REL_EXTENT) + self.REL_BIAS_QK_SCALE = gl.constexpr(REL_BIAS_QK_SCALE) + self.IS_FP8 = gl.constexpr(IS_FP8) + self.q_strides = q_strides + self.rel_strides = rel_strides + self.k_strides = k_strides + self.v_strides = v_strides + self.qk_layout = gl.constexpr(qk_layout) + self.pv_layout = gl.constexpr(pv_layout) + # qk_kw is derived from a 128-bit load / dtype bitwidth. + # pv_kw is empirically tuned. + qk_kw = 16 if IS_FP8 else 8 + pv_kw = 16 if IS_FP8 else 4 + self.q_layout = gl.constexpr(gl.DotOperandLayout(0, qk_layout, k_width=qk_kw)) + self.k_layout = gl.constexpr(gl.DotOperandLayout(1, qk_layout, k_width=qk_kw)) + self.p_layout = gl.constexpr(gl.DotOperandLayout(0, pv_layout, k_width=pv_kw)) + self.v_layout = gl.constexpr(gl.DotOperandLayout(1, pv_layout, k_width=pv_kw)) + self.load_layout = gl.constexpr(load_layout) + self.store_layout = gl.constexpr(store_layout) + self.k_smem_layout = gl.constexpr(k_smem_layout) + self.v_smem_layout = gl.constexpr(v_smem_layout) + + +@aggregate +class ExtendProgram: + cfg: gl.constexpr + q_ptr: gl.tensor + rel_logits_ptr: gl.tensor + k_cache_ptr: gl.tensor + v_cache_ptr: gl.tensor + page_table_ptr: gl.tensor + output_ptr: gl.tensor + lse_ptr: gl.tensor + batch: gl.tensor + q_head: gl.tensor + kv_head: gl.tensor + q_start: gl.tensor + seq_base: gl.tensor + seq_len: gl.tensor + prefix: gl.tensor + cache_len: gl.tensor + + @gluon.constexpr_function + def __init__( + self, + cfg, + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + output_ptr, + lse_ptr, + batch, + q_head, + kv_head, + q_start, + seq_base, + seq_len, + prefix, + cache_len, + ): + self.cfg = gl.constexpr(cfg) + self.q_ptr = q_ptr + self.rel_logits_ptr = rel_logits_ptr + self.k_cache_ptr = k_cache_ptr + self.v_cache_ptr = v_cache_ptr + self.page_table_ptr = page_table_ptr + self.output_ptr = output_ptr + self.lse_ptr = lse_ptr + self.batch = batch + self.q_head = q_head + self.kv_head = kv_head + self.q_start = q_start + self.seq_base = seq_base + self.seq_len = seq_len + self.prefix = prefix + self.cache_len = cache_len + + @gluon.jit + def create( + cfg, + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + output_ptr, + lse_ptr, + cu_seqlens_q_ptr, + cache_seqlens_ptr, + ): + # Gluon treats ``cfg`` as the constexpr factory argument, while mypy + # models the first parameter of this aggregate method as ``self``. + block_in_req = gl.program_id(0) + batch = gl.program_id(1) + q_head = gl.program_id(2) + q_start = block_in_req * cfg.BLOCK_M # type: ignore[attr-defined] + seq_base = gl.load(cu_seqlens_q_ptr + batch) + seq_end = gl.load(cu_seqlens_q_ptr + batch + 1) + seq_len = seq_end - seq_base + cache_len = gl.load(cache_seqlens_ptr + batch) + prefix = cache_len - seq_len + kv_head = q_head // cfg.GROUP_SIZE # type: ignore[attr-defined] + return ExtendProgram( + gl.constexpr(cfg), + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + output_ptr, + lse_ptr, + batch, + q_head, + kv_head, + q_start, + seq_base, + seq_len, + prefix, + cache_len, + ) + + @gluon.jit + def load_q(self): + cfg = self.cfg + offs_m = self.q_start + gl.arange( + 0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.q_layout) + ) + offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.q_layout)) + row = self.seq_base + offs_m + offsets = cfg.q_strides.offsets(row[:, None], self.q_head, offs_d[None, :]) + mask = offs_m[:, None] < self.seq_len + return cdna4.buffer_load(self.q_ptr, offsets, mask=mask, other=0.0) + + @gluon.jit + def load_page(self, start_n): + cfg = self.cfg + page_index = start_n // cfg.PAGE_SIZE + valid = start_n < self.cache_len + return gl.load( + self.page_table_ptr + self.batch * cfg.PAGE_TABLE_STRIDE + page_index, + mask=valid, + other=0, + ) + + @gluon.jit + def issue_load_k(self, physical_page, k_smem): + cfg = self.cfg + offs_n = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(1, cfg.load_layout)) + offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.load_layout)) + offsets = cfg.k_strides.offsets( + physical_page, + offs_n[:, None], + self.kv_head, + offs_d[None, :], + ) + async_copy.global_load_to_shared(k_smem, self.k_cache_ptr + offsets) + async_copy.commit_group() + + @gluon.jit + def issue_load_v(self, physical_page, v_smem): + cfg = self.cfg + offs_n = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(1, cfg.load_layout)) + offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.load_layout)) + offsets = cfg.v_strides.offsets( + physical_page, + offs_n[:, None], + self.kv_head, + offs_d[None, :], + ) + async_copy.global_load_to_shared(v_smem, self.v_cache_ptr + offsets) + async_copy.commit_group() + + @gluon.jit + def shared_load_k(self, k_smem): + cfg = self.cfg + k_buffer = k_smem.permute([1, 0]) + return k_buffer.load(cfg.k_layout) + + @gluon.jit + def shared_load_v(self, v_smem): + cfg = self.cfg + return v_smem.load(cfg.v_layout) + + @gluon.jit + def compute_qk(self, q, k): + cfg = self.cfg + qk = gl.zeros( + [cfg.BLOCK_M, cfg.BLOCK_N], dtype=gl.float32, layout=cfg.qk_layout + ) + return cdna4.mfma(q, k, qk) + + @gluon.jit + def apply_rel_bias(self, qk, start_n): + cfg = self.cfg + offs_m = self.q_start + gl.arange( + 0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.qk_layout) + ) + offs_n_abs = start_n + gl.arange( + 0, cfg.BLOCK_N, layout=gl.SliceLayout(0, cfg.qk_layout) + ) + q_pos = self.prefix + offs_m + rel_dist = q_pos[:, None] - offs_n_abs[None, :] + rel_valid = (rel_dist >= 0) & (rel_dist < cfg.REL_EXTENT) + rel_idx = gl.where(rel_dist >= 0, rel_dist, 0) + rel_idx = gl.where(rel_idx < cfg.REL_EXTENT, rel_idx, cfg.REL_EXTENT - 1) + offsets = cfg.rel_strides.offsets( + self.seq_base + offs_m[:, None], self.q_head, rel_idx + ) + mask = (offs_m[:, None] < self.seq_len) & rel_valid + rel_bias = cdna4.buffer_load( + self.rel_logits_ptr, offsets, mask=mask, other=0.0 + ).to(gl.float32) + return qk + rel_bias * cfg.REL_BIAS_QK_SCALE + + @gluon.jit + def compute_pv(self, p, v, acc): + return cdna4.mfma(p, v, acc) + + @gluon.jit + def init_attention_state(self): + cfg = self.cfg + m_i = gl.full( + [cfg.BLOCK_M], + value=-float("inf"), + dtype=gl.float32, + layout=gl.SliceLayout(1, cfg.pv_layout), + ) + l_i = gl.full( + [cfg.BLOCK_M], + value=0, + dtype=gl.float32, + layout=gl.SliceLayout(1, cfg.pv_layout), + ) + acc = gl.zeros( + [cfg.BLOCK_M, cfg.HEAD_DIM], dtype=gl.float32, layout=cfg.pv_layout + ) + return m_i, l_i, acc + + @gluon.jit + def softmax(self, qk, m_i, l_i, acc): + cfg = self.cfg + # In sliding window case, some rows can see fully masked tiles before + # any valid KV. Guard the online softmax state so `-inf - -inf` does not + # produce NaNs. + HAS_INVALID: gl.constexpr = cfg.WINDOW_LEFT >= 0 + + row_max = max(qk, 1) + m_new = maximum(m_i, row_max) + m_new_scaled = m_new * cfg.SM_SCALE + if HAS_INVALID: + invalid = m_new == -float("inf") + m_new_scaled = gl.where(invalid, 0.0, m_new_scaled) + + qk_shifted = qk * cfg.SM_SCALE - m_new_scaled[:, None] + p = gl.exp2(qk_shifted) + m_diff = m_i * cfg.SM_SCALE - m_new_scaled + if HAS_INVALID: + m_diff = gl.where(invalid, 0.0, m_diff) + + alpha = gl.exp2(m_diff) + l_ij = gl.sum(p, axis=1) + l_i = l_i * alpha + l_ij + acc = acc * alpha[:, None] + p = p.to(self.q_ptr.dtype.element_ty) + p = gl.convert_layout(p, cfg.p_layout) + return p, m_new, l_i, acc + + @gluon.jit + def store_output(self, output): + cfg = self.cfg + offs_m = self.q_start + gl.arange( + 0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.store_layout) + ) + offs_d = gl.arange(0, cfg.HEAD_DIM, layout=gl.SliceLayout(0, cfg.store_layout)) + offsets = ( + ((self.seq_base + offs_m[:, None]) * cfg.N_HEADS + self.q_head) + * cfg.HEAD_DIM + + offs_d[None, :] + ).to(gl.int32) + mask = offs_m[:, None] < self.seq_len + output = output.to(self.output_ptr.dtype.element_ty) + cdna4.buffer_store(output, self.output_ptr, offsets, mask=mask) + + @gluon.jit + def store_lse(self, l_i, m_i): + cfg = self.cfg + if cfg.HAS_LSE: + offs_m = self.q_start + gl.arange( + 0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.pv_layout) + ) + offsets = ((self.seq_base + offs_m) * cfg.N_HEADS + self.q_head).to( + gl.int32 + ) + mask = offs_m < self.seq_len + lse_l_i = gl.where(l_i > 0.0, l_i, 1.0) + # Softmax runs in base-2 (exp2 hardware fast path), so m_i*SM_SCALE + + # log2(l_i) is the LSE in base-2 units. Convert to natural log (the + # public op contract / torch.logsumexp convention) by scaling by ln2. + lse = (m_i * cfg.SM_SCALE + gl.log2(lse_l_i)) * _LN2 + cdna4.buffer_store(lse, self.lse_ptr, offsets, mask=mask) + + +@gluon.jit +def _rel_mha_extend_fp16( + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + output_ptr, + lse_ptr, + cu_seqlens_q_ptr, + cache_seqlens_ptr, + Q_STRIDE_T: gl.constexpr, + Q_STRIDE_H: gl.constexpr, + Q_STRIDE_D: gl.constexpr, + K_STRIDE_B: gl.constexpr, + K_STRIDE_P: gl.constexpr, + K_STRIDE_H: gl.constexpr, + K_STRIDE_D: gl.constexpr, + V_STRIDE_B: gl.constexpr, + V_STRIDE_P: gl.constexpr, + V_STRIDE_H: gl.constexpr, + V_STRIDE_D: gl.constexpr, + SM_SCALE: gl.constexpr, + PAGE_TABLE_STRIDE: gl.constexpr, + PAGE_SIZE: gl.constexpr, + N_HEADS: gl.constexpr, + N_KV_HEADS: gl.constexpr, + HEAD_DIM: gl.constexpr, + BLOCK_M: gl.constexpr, + BLOCK_N: gl.constexpr, + NUM_WARPS: gl.constexpr, + IS_CAUSAL: gl.constexpr, + HAS_LSE: gl.constexpr, + WINDOW_LEFT: gl.constexpr, + REL_STRIDE_T: gl.constexpr, + REL_STRIDE_H: gl.constexpr, + REL_STRIDE_E: gl.constexpr, + REL_EXTENT: gl.constexpr, + REL_BIAS_QK_SCALE: gl.constexpr, + IS_FP8: gl.constexpr, +): + cfg = ExtendConfig( + N_HEADS, + N_KV_HEADS, + HEAD_DIM, + SM_SCALE, + BLOCK_M, + BLOCK_N, + NUM_WARPS, + PAGE_SIZE, + PAGE_TABLE_STRIDE, + IS_CAUSAL, + HAS_LSE, + WINDOW_LEFT, + REL_EXTENT, + REL_BIAS_QK_SCALE, + IS_FP8, + InputStrides(Q_STRIDE_T, Q_STRIDE_H, Q_STRIDE_D), + InputStrides(REL_STRIDE_T, REL_STRIDE_H, REL_STRIDE_E), + PagedKVStrides(K_STRIDE_B, K_STRIDE_P, K_STRIDE_H, K_STRIDE_D), + PagedKVStrides(V_STRIDE_B, V_STRIDE_P, V_STRIDE_H, V_STRIDE_D), + ) + program = ExtendProgram.create( + cfg, + q_ptr, + rel_logits_ptr, + k_cache_ptr, + v_cache_ptr, + page_table_ptr, + output_ptr, + lse_ptr, + cu_seqlens_q_ptr, + cache_seqlens_ptr, + ) + # Over-provisioned tile past this request's real query rows: nothing to do. + if program.q_start >= program.seq_len: + return + k_smem = gl.allocate_shared_memory( + k_cache_ptr.dtype.element_ty, [cfg.BLOCK_N, cfg.HEAD_DIM], cfg.k_smem_layout + ) + v_smem = gl.allocate_shared_memory( + v_cache_ptr.dtype.element_ty, [cfg.BLOCK_N, cfg.HEAD_DIM], cfg.v_smem_layout + ) + + q = program.load_q() + m_i, l_i, acc = program.init_attention_state() + + offs_m_q = program.q_start + gl.arange( + 0, cfg.BLOCK_M, layout=gl.SliceLayout(1, cfg.qk_layout) + ) + offs_n_q = gl.arange(0, cfg.BLOCK_N, layout=gl.SliceLayout(0, cfg.qk_layout)) + diag_row = program.prefix + offs_m_q + + if IS_CAUSAL: + kv_end = min(program.cache_len, program.prefix + program.q_start + cfg.BLOCK_M) + else: + kv_end = program.cache_len + + # Sliding window (inclusive-left, matches flash-attn window_size=(W, 0)): + # skip KV tiles entirely below the window's lower edge. The tile's top query + # row sits at absolute position pos_top; its window opens at + # pos_top - WINDOW_LEFT (that key is visible -> W + 1 keys total). The min() + # form clamps to 0 without the shadowed builtin max(). + if cfg.WINDOW_LEFT >= 0: + pos_top = program.prefix + program.q_start + kv_start = pos_top - min(pos_top, cfg.WINDOW_LEFT) + kv_start = (kv_start // cfg.BLOCK_N) * cfg.BLOCK_N + else: + kv_start = 0 + + for start_n in range(kv_start, kv_end, cfg.BLOCK_N): + physical_page = program.load_page(start_n) + program.issue_load_k(physical_page, k_smem) + program.issue_load_v(physical_page, v_smem) + + async_copy.wait_group(1) + k = program.shared_load_k(k_smem) + qk = program.compute_qk(q, k) + qk = program.apply_rel_bias(qk, start_n) + + if cfg.WINDOW_LEFT >= 0: + # Window lower edge + cache bound always apply; causal upper edge only + # when IS_CAUSAL (independent layering keeps non-causal + window correct). + offs_n_abs = start_n + offs_n_q + mask = (offs_n_abs[None, :] >= diag_row[:, None] - cfg.WINDOW_LEFT) & ( + offs_n_abs[None, :] < program.cache_len + ) + if IS_CAUSAL: + mask &= offs_n_abs[None, :] <= diag_row[:, None] + qk = gl.where(mask, qk, -float("inf")) + elif IS_CAUSAL: + if start_n + cfg.BLOCK_N > program.prefix + program.q_start: + offs_n_abs = start_n + offs_n_q + mask = (offs_n_abs[None, :] <= diag_row[:, None]) & ( + offs_n_abs[None, :] < program.cache_len + ) + qk = gl.where(mask, qk, -float("inf")) + else: + if start_n + cfg.BLOCK_N > program.cache_len: + offs_n_abs = start_n + offs_n_q + qk = gl.where( + offs_n_abs[None, :] < program.cache_len, qk, -float("inf") + ) + + p, m_i, l_i, acc = program.softmax(qk, m_i, l_i, acc) + + async_copy.wait_group(0) + v = program.shared_load_v(v_smem) + acc = program.compute_pv(p, v, acc) + + denom = gl.where(l_i > 0.0, l_i, 1.0) + output = acc * (1.0 / denom)[:, None] + output = gl.convert_layout(output, cfg.store_layout) + program.store_output(output) + program.store_lse(l_i, m_i) + + +def gluon_rel_mha_extend_gfx950( + q: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_kv: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + page_table: torch.Tensor, + cache_seqlens: torch.Tensor, + window_left: int = -1, + return_lse: bool = False, + max_seqlen_q: int = 1, + max_seqlen_k: int = 1, + rel_logits: torch.Tensor | None = None, + softmax_scale: float | None = None, + q_scale: torch.Tensor | None = None, + k_scale: torch.Tensor | None = None, + v_scale: torch.Tensor | None = None, + out: torch.Tensor | None = None, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + assert rel_logits is not None + head_dim = q.shape[2] + n_heads = q.shape[1] + n_kv_heads = k_cache.shape[2] + page_size = k_cache.shape[1] + block_m, block_n, num_warps = _select_extend_tile(max_seqlen_q, page_size) + if softmax_scale is None: + softmax_scale = 1.0 / math.sqrt(head_dim) + sm_scale = softmax_scale * _INV_LN2_VALUE + rel_bias_qk_scale = 1.0 / softmax_scale + + # max_seqlen_q must be >= the true max query length; extra tiles early-exit. + batch = cu_seqlens_q.shape[0] - 1 + safe_max_q = max_seqlen_q if max_seqlen_q > 0 else 1 + blocks_per_req = (safe_max_q + block_m - 1) // block_m + cu_q_i32 = cu_seqlens_q.to(torch.int32).contiguous() + cache_i32 = cache_seqlens.to(torch.int32).contiguous() + + is_fp8 = q.dtype in (torch.float8_e4m3fn, torch.float8_e5m2) + out_dtype = torch.bfloat16 if is_fp8 else q.dtype + if page_size not in (64, 128, 256): + raise ValueError( + "gfx950 Gluon relative-attention extend requires page size " + f"64, 128, or 256; got {page_size}" + ) + output = ( + torch.empty(q.shape, device=q.device, dtype=out_dtype) if out is None else out + ) + if return_lse: + lse = torch.empty((q.shape[0], n_heads), device=q.device, dtype=torch.float32) + lse_arg = lse + else: + lse = None + lse_arg = q + grid = (blocks_per_req, batch, n_heads) + _rel_mha_extend_fp16[grid]( + q, + rel_logits, + k_cache, + v_cache, + page_table, + output, + lse_arg, + cu_q_i32, + cache_i32, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + k_cache.stride(3), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + v_cache.stride(3), + sm_scale, + page_table.stride(0), + page_size, + n_heads, + n_kv_heads, + head_dim, + block_m, + block_n, + num_warps, + True, + return_lse, + window_left, + rel_logits.stride(0), + rel_logits.stride(1), + rel_logits.stride(2), + rel_logits.shape[2], + rel_bias_qk_scale, + is_fp8, + num_warps=num_warps, + ) + if return_lse: + assert lse is not None + return output, lse + return output diff --git a/vllm/models/inkling/amd/ops/gluon/utils.py b/vllm/models/inkling/amd/ops/gluon/utils.py new file mode 100644 index 0000000000..90f2e586bc --- /dev/null +++ b/vllm/models/inkling/amd/ops/gluon/utils.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Copyright (c) 2026 LightSeek Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from vllm.triton_utils import aggregate, gl, gluon, tl + +_INV_LN2_VALUE = 1.4426950408889634 +_INV_LN2 = tl.constexpr(_INV_LN2_VALUE) +_LN2_VALUE = 0.6931471805599453 +_LN2 = tl.constexpr(_LN2_VALUE) +_PROPAGATE_NAN_ALL = gl.constexpr(tl.PropagateNan.ALL) + + +@gluon.jit +def maximum(a, b, propagate_nan: gl.constexpr = _PROPAGATE_NAN_ALL): + return gl.maximum(a, b, propagate_nan=propagate_nan) + + +@gluon.jit +def max(input, axis=None, keep_dims=False): + return gl.reduce(input, axis, maximum, keep_dims=keep_dims) + + +@gluon.constexpr_function +def attention_layouts(head_dim, block_n, is_fp8, dtype, num_warps, instr_shape): + mfma = gl.amd.AMDMFMALayout( + version=4, + instr_shape=instr_shape, + transposed=True, + warps_per_cta=[num_warps, 1], + ) + qk_layout = mfma + pv_layout = mfma + # qk_kw is derived from a 128-bit load / dtype bitwidth; pv_kw is tuned. + qk_kw = 16 if is_fp8 else 8 + pv_kw = 8 if is_fp8 else 4 + q_layout = gl.DotOperandLayout(0, qk_layout, k_width=qk_kw) + k_layout = gl.DotOperandLayout(1, qk_layout, k_width=qk_kw) + p_layout = gl.DotOperandLayout(0, pv_layout, k_width=pv_kw) + v_layout = gl.DotOperandLayout(1, pv_layout, k_width=pv_kw) + # load_vec = elems/lane (dtype-dependent, == qk_kw); load_threads span HEAD_DIM. + load_vec = 16 if is_fp8 else 8 + load_threads = head_dim // load_vec + load_layout = gl.BlockedLayout( + [1, load_vec], [64 // load_threads, load_threads], [num_warps, 1], [1, 0] + ) + # store_vec is always 16-bit (128 / 16 = 8) regardless of input dtype. + store_vec = 8 + store_threads = head_dim // store_vec + store_layout = gl.BlockedLayout( + [1, store_vec], [64 // store_threads, store_threads], [num_warps, 1], [1, 0] + ) + # Take only the built-in's padding, not its swizzle: the swizzle scatters + # head_dim across banks, making the LDS write stride non-constant, so it can't + # lower through async_copy's affine [1, 0] load. Padding-only is affine and + # DMA-legal. + # TODO(perf): to also use the swizzle, co-design a matched load layout so the + # DMA stays legal. + k_api = gl.amd.cdna4.compute_efficient_padded_shared_layout( + k_layout, [block_n, head_dim], dtype, is_k_contig=True + ) + v_api = gl.amd.cdna4.compute_efficient_padded_shared_layout( + v_layout, [block_n, head_dim], dtype, is_k_contig=False + ) + assert k_api is not None and v_api is not None, ( + "no CDNA4 padded shared layout for this operand/dtype" + ) + k_pairs = list(k_api.interval_padding_pairs) + v_pairs = list(v_api.interval_padding_pairs) + assert len(k_pairs) == 1 and len(v_pairs) == 1, ( + "expected a single interval padding pair from the built-in" + ) + k_smem_layout = gl.PaddedSharedLayout.with_identity_for( + [[int(k_pairs[0][0]), int(k_pairs[0][1])]], [block_n, head_dim], [1, 0] + ) + v_smem_layout = gl.PaddedSharedLayout.with_identity_for( + [[int(v_pairs[0][0]), int(v_pairs[0][1])]], [block_n, head_dim], [1, 0] + ) + return ( + qk_layout, + pv_layout, + q_layout, + k_layout, + p_layout, + v_layout, + load_layout, + store_layout, + k_smem_layout, + v_smem_layout, + ) + + +@aggregate +class InputStrides: + stride_t: gl.constexpr + stride_h: gl.constexpr + stride_d: gl.constexpr + + @gluon.constexpr_function + def __init__(self, stride_t, stride_h, stride_d): + self.stride_t = gl.constexpr(stride_t) + self.stride_h = gl.constexpr(stride_h) + self.stride_d = gl.constexpr(stride_d) + + @gluon.jit + def offsets(self, token, head, dim): + return (token * self.stride_t + head * self.stride_h + dim * self.stride_d).to( + gl.int32 + ) + + +@aggregate +class PagedKVStrides: + stride_b: gl.constexpr + stride_p: gl.constexpr + stride_h: gl.constexpr + stride_d: gl.constexpr + + @gluon.constexpr_function + def __init__(self, stride_b, stride_p, stride_h, stride_d): + self.stride_b = gl.constexpr(stride_b) + self.stride_p = gl.constexpr(stride_p) + self.stride_h = gl.constexpr(stride_h) + self.stride_d = gl.constexpr(stride_d) + + @gluon.jit + def offsets(self, page, token, head, dim): + # KV pools can exceed the 32-bit buffer-offset range. Keep the page + # arithmetic in int64, matching the original kernel's large-cache path. + return ( + page.to(gl.int64) * self.stride_b + + token.to(gl.int64) * self.stride_p + + head * self.stride_h + + dim * self.stride_d + ) diff --git a/vllm/models/inkling/amd/ops/lamport.py b/vllm/models/inkling/amd/ops/lamport.py new file mode 100644 index 0000000000..9705a9a3df --- /dev/null +++ b/vllm/models/inkling/amd/ops/lamport.py @@ -0,0 +1,766 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Deadlock-free fused RS + short-conv + AG + residual + RMSNorm. + +The public integration surface is ``LamportRSConv.rs_sconv_ag_add_norm``. + +Liveness +-------- +Large grids are deliberately split at the two communication dependencies: + + 1. ``_publish_input_kernel`` only publishes rank partials. + 2. ``_reduce_insert_kernel`` waits, reduces, and inserts the local shard. + 3. ``_sconv_publish_kernel`` only computes and publishes the local result. + 4. ``_gather_norm_kernel`` waits, gathers, and normalizes. + +Without PDL, CUDA stream order completes a producer before its consumer. With +PDL, every producer CTA posts all peer stores before triggering its dependent, +and the consumer executes ``gdc_wait`` before polling. A consumer therefore +waits only for stores from a producer that is already running (or complete) on +another GPU. It never waits for another block in its own grid. Consequently +spinning consumers cannot occupy resources needed by any producer, and the +wait-for graph has no cycle. The proof is independent of grid size, block +dispatch order, and occupancy. + +For one token, the first three phases use eight independent channel slices to +expose enough CTA parallelism for decode latency. Each slice has exclusive +ownership of its cache and Lamport columns; the gather/RMSNorm phase retains +one CTA per token so no cross-CTA reduction or completion counter is needed. + +Immediate buffer reuse (including replay of a captured CUDA graph) is also +safe. Rank R cannot republish input for call n+1 until its gather for n has +finished; that gather waited for owner O's output, which O publishes only +after consuming R's input for n. Likewise, R cannot republish output for n+1 +until its reduction for n+1 has observed destination D's input; D publishes +that input only after its gather consumed R's output for n. Thus every prior +read happens-before a same-slot rewrite. Three generations reduce incidental +coupling for ordinary launches, but correctness does not depend on rotation. + +The payload itself is the Lamport flag. A 32-bit store publishes two bf16s +atomically; 0x80008000 (two negative zeroes) denotes an empty pair. Real +negative zeroes are changed to positive zero before publication. Consumers +use volatile 32-bit loads and restore the sentinel after consuming a slot. +""" + +from __future__ import annotations + +import os + +import torch + +from vllm.distributed import get_tp_group +from vllm.distributed.parallel_state import in_the_same_node_as +from vllm.logger import init_logger +from vllm.triton_utils import tl, triton + +logger = init_logger(__name__) + +_MAX_TOKENS = 16384 +_EMPTY_PAIR = tl.constexpr(0x80008000) + + +@triton.jit +def _pack_bf16_pairs(values): + """Pack bf16 values into atomic u32 pairs and reserve negative zero.""" + lo, hi = tl.split(values.reshape([values.shape[0] // 2, 2])) + lo = lo.to(tl.uint16, bitcast=True) + hi = hi.to(tl.uint16, bitcast=True) + lo = tl.where(lo == 0x8000, 0, lo).to(tl.uint32) + hi = tl.where(hi == 0x8000, 0, hi).to(tl.uint32) + return lo | (hi << 16) + + +@triton.jit +def _unpack_bf16_pairs(values): + lo = (values & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True) + hi = (values >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True) + return tl.interleave(lo, hi) + + +@triton.jit +def _wait_pairs(ptr, offsets, mask): + values = tl.load(ptr + offsets, mask=mask, other=0, volatile=True) + while tl.max(tl.where(mask & (values == _EMPTY_PAIR), 1, 0)) != 0: + values = tl.load(ptr + offsets, mask=mask, other=0, volatile=True) + return values + + +@triton.jit +def _publish_input_kernel( + stage_ptr, + peer_ptrs, + peer_offset_u32, + stride_stage_t, + C: tl.constexpr, + CS: tl.constexpr, + CS_P2: tl.constexpr, + SPLITS: tl.constexpr, + RANK: tl.constexpr, + WORLD: tl.constexpr, + USE_PDL: tl.constexpr, + launch_pdl: tl.constexpr, +): + """Publish this rank's full partial row into every shard owner's slots.""" + token = tl.program_id(0).to(tl.int64) + split = tl.program_id(1).to(tl.int64) + CSS: tl.constexpr = CS // SPLITS + pair = tl.arange(0, CS_P2 // 2) + pair_mask = pair < CSS // 2 + elem = tl.arange(0, CS_P2) + elem_mask = elem < CSS + ptrs = peer_ptrs.to(tl.pointer_type(tl.uint64)) + # Address generation is independent of the preceding stream kernel. The + # acquire remains before stage/peer loads, which may consume its writes. + if USE_PDL: + tl.extra.cuda.gdc_wait() + + for owner in tl.static_range(WORLD): + values = tl.load( + stage_ptr + token * stride_stage_t + owner * CS + split * CSS + elem, + mask=elem_mask, + other=0.0, + ) + packed = _pack_bf16_pairs(values) + base = tl.load(ptrs + owner).to(tl.pointer_type(tl.uint32)) + dst = (token * WORLD + RANK) * (CS // 2) + split * (CSS // 2) + pair + tl.store(base + peer_offset_u32 + dst, packed, mask=pair_mask) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +@triton.jit +def _reduce_insert_kernel( + input_peer_ptrs, + input_peer_offset_u32, + cache_ptr, + slot_ptr, + stride_cache_block, + stride_cache_head, + stride_cache_token, + stride_cache_dim, + block_size, + C: tl.constexpr, + CS: tl.constexpr, + CS_P2: tl.constexpr, + SPLITS: tl.constexpr, + HEAD_SIZE: tl.constexpr, + CACHE_OFFSET: tl.constexpr, + RANK: tl.constexpr, + WORLD: tl.constexpr, + USE_PDL: tl.constexpr, + launch_pdl: tl.constexpr, +): + """Consume all partials for this rank and insert the reduced cache row.""" + token = tl.program_id(0).to(tl.int64) + split = tl.program_id(1).to(tl.int64) + CSS: tl.constexpr = CS // SPLITS + source = tl.arange(0, WORLD) + pair = tl.arange(0, CS_P2 // 2) + pair_mask = pair < CSS // 2 + offsets = ( + (token * WORLD + source)[:, None] * (CS // 2) + + split * (CSS // 2) + + pair[None, :] + ) + mask = tl.full([WORLD], True, tl.int1)[:, None] & pair_mask[None, :] + input_ptrs = input_peer_ptrs.to(tl.pointer_type(tl.uint64)) + input_u32 = tl.load(input_ptrs + RANK).to(tl.pointer_type(tl.uint32)) + input_u32 += input_peer_offset_u32 + # Slot metadata and the cache destination do not depend on input publish. + slot = tl.load(slot_ptr + token) + valid = slot >= 0 + channel = tl.arange(0, CS_P2) + channel_mask = channel < CSS + global_channel = split * CSS + channel + head = tl.minimum(global_channel // HEAD_SIZE, CS // HEAD_SIZE - 1) + dim = CACHE_OFFSET + global_channel % HEAD_SIZE + safe_slot = tl.maximum(slot, 0).to(tl.int64) + dst = ( + cache_ptr + + (safe_slot // block_size) * stride_cache_block + + head * stride_cache_head + + (safe_slot % block_size) * stride_cache_token + + dim * stride_cache_dim + ) + if USE_PDL: + tl.extra.cuda.gdc_wait() + packed = _wait_pairs(input_u32, offsets, mask) + + lo = (packed & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True) + hi = (packed >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True) + reduced = tl.interleave( + tl.sum(lo.to(tl.float32), axis=0).to(tl.bfloat16), + tl.sum(hi.to(tl.float32), axis=0).to(tl.bfloat16), + ) + tl.store( + input_u32 + offsets, + tl.full([WORLD, CS_P2 // 2], _EMPTY_PAIR, tl.uint32), + mask=mask, + ) + + tl.store(dst, reduced, mask=valid & channel_mask) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +@triton.jit +def _sconv_publish_kernel( + peer_ptrs, + peer_offset_u32, + residual_ptr, + weight_ptr, + cache_ptr, + position_ptr, + sequence_ptr, + slot_ptr, + block_table_ptr, + stride_residual_t, + stride_cache_block, + stride_cache_head, + stride_cache_token, + stride_cache_dim, + stride_block_table_r, + max_blocks, + block_size, + C: tl.constexpr, + CS: tl.constexpr, + CS_P2: tl.constexpr, + SPLITS: tl.constexpr, + HEAD_SIZE: tl.constexpr, + CACHE_OFFSET: tl.constexpr, + RANK: tl.constexpr, + WORLD: tl.constexpr, + WINDOW: tl.constexpr, + USE_PDL: tl.constexpr, + launch_pdl: tl.constexpr, +): + """Compute this rank's shard, then publish it to every rank.""" + tl.static_assert(WINDOW == 4) + token = tl.program_id(0).to(tl.int64) + split = tl.program_id(1).to(tl.int64) + CSS: tl.constexpr = CS // SPLITS + channel = tl.arange(0, CS_P2) + channel_mask = channel < CSS + global_channel = split * CSS + channel + head = tl.minimum(global_channel // HEAD_SIZE, CS // HEAD_SIZE - 1) + dim = CACHE_OFFSET + global_channel % HEAD_SIZE + slot = tl.load(slot_ptr + token) + valid = slot >= 0 + position = tl.load(position_ptr + token) + sequence = tl.load(sequence_ptr + token) + safe_slot = tl.maximum(slot, 0).to(tl.int64) + own_ptr = ( + cache_ptr + + (safe_slot // block_size) * stride_cache_block + + head * stride_cache_head + + (safe_slot % block_size) * stride_cache_token + + dim * stride_cache_dim + ) + # These loads were made visible before reduce/insert could signal us and + # are independent of its cache store. Hoisting them gives PDL useful work + # to overlap while retaining the acquire before every cache read. + residual = tl.load( + residual_ptr + token * stride_residual_t + RANK * CS + global_channel, + mask=channel_mask, + other=0.0, + ) + weight0 = tl.load( + weight_ptr + global_channel * WINDOW, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + weight1 = tl.load( + weight_ptr + global_channel * WINDOW + 1, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + weight2 = tl.load( + weight_ptr + global_channel * WINDOW + 2, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + weight3 = tl.load( + weight_ptr + global_channel * WINDOW + 3, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + ptrs = peer_ptrs.to(tl.pointer_type(tl.uint64)) + if USE_PDL: + tl.extra.cuda.gdc_wait() + current = tl.load(own_ptr, mask=valid & channel_mask, other=0.0) + + conv = tl.zeros([CS_P2], tl.float32) + for tap_idx in tl.static_range(WINDOW): + source_position = position - (WINDOW - 1) + tap_idx + take = valid & (source_position >= 0) + if tap_idx == WINDOW - 1: + value = tl.where(take, current.to(tl.float32), 0.0) + else: + safe_position = tl.maximum(source_position, 0) + logical_block = tl.minimum(safe_position // block_size, max_blocks - 1) + physical_block = tl.load( + block_table_ptr + sequence * stride_block_table_r + logical_block, + mask=take, + other=0, + ).to(tl.int64) + source_ptr = ( + cache_ptr + + physical_block * stride_cache_block + + head * stride_cache_head + + (safe_position % block_size) * stride_cache_token + + dim * stride_cache_dim + ) + cached = tl.load(source_ptr, mask=take & channel_mask, other=0.0) + value = tl.where(take, cached.to(tl.float32), 0.0) + if tap_idx == 0: + weight = weight0 + elif tap_idx == 1: + weight = weight1 + elif tap_idx == 2: + weight = weight2 + else: + weight = weight3 + conv += value * weight + + # Preserve both bf16 rounding points of the original sublayer. + short_conv_with_skip = (conv + current.to(tl.float32)).to(tl.bfloat16) + output = (residual.to(tl.float32) + short_conv_with_skip.to(tl.float32)).to( + tl.bfloat16 + ) + + pair = tl.arange(0, CS_P2 // 2) + pair_mask = pair < CSS // 2 + packed = _pack_bf16_pairs(output) + row_offset = token * (C // 2) + RANK * (CS // 2) + split * (CSS // 2) + pair + + for destination in tl.static_range(WORLD): + base = tl.load(ptrs + destination).to(tl.pointer_type(tl.uint32)) + tl.store(base + peer_offset_u32 + row_offset, packed, mask=pair_mask) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +@triton.jit +def _gather_norm_kernel( + output_peer_ptrs, + output_peer_offset_u32, + norm_weight_ptr, + normed_ptr, + residual_out_ptr, + eps, + stride_output_t, + C: tl.constexpr, + C_P2: tl.constexpr, + RANK: tl.constexpr, + HAS_NORM: tl.constexpr, + USE_PDL: tl.constexpr, + launch_pdl: tl.constexpr, +): + """Consume a complete row; one CTA owns all outputs for one token.""" + token = tl.program_id(0).to(tl.int64) + pair = tl.arange(0, C_P2 // 2) + pair_mask = pair < C // 2 + output_ptrs = output_peer_ptrs.to(tl.pointer_type(tl.uint64)) + output_u32 = tl.load(output_ptrs + RANK).to(tl.pointer_type(tl.uint32)) + output_u32 += output_peer_offset_u32 + offsets = token * (C // 2) + pair + channel = tl.arange(0, C_P2) + channel_mask = channel < C + # Gamma is independent of the preceding sconv publication. Keep the + # acquire immediately before polling the Lamport output slots. + if HAS_NORM: + weight = tl.load(norm_weight_ptr + channel, mask=channel_mask, other=0.0) + if USE_PDL: + tl.extra.cuda.gdc_wait() + packed = _wait_pairs(output_u32, offsets, pair_mask) + row = _unpack_bf16_pairs(packed) + tl.store( + residual_out_ptr + token * stride_output_t + channel, row, mask=channel_mask + ) + if HAS_NORM: + row_f32 = tl.where(channel_mask, row.to(tl.float32), 0.0) + inv_rms = tl.rsqrt(tl.sum(row_f32 * row_f32, axis=0) / C + eps) + tl.store( + normed_ptr + token * stride_output_t + channel, + (row_f32 * inv_rms * weight.to(tl.float32)).to(tl.bfloat16), + mask=channel_mask, + ) + tl.store( + output_u32 + offsets, + tl.full([C_P2 // 2], _EMPTY_PAIR, tl.uint32), + mask=pair_mask, + ) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +@triton.jit +def _validate_lamport_init_kernel( + input_peer_ptrs, + output_peer_ptrs, + bad_ptr, + num_pairs, + RANK: tl.constexpr, + BLOCK: tl.constexpr, +): + """Validate both complete local allocations through their fabric pointers.""" + offsets = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK) + mask = offsets < num_pairs + ptrs_in = input_peer_ptrs.to(tl.pointer_type(tl.uint64)) + ptrs_out = output_peer_ptrs.to(tl.pointer_type(tl.uint64)) + local_in = tl.load(ptrs_in + RANK).to(tl.pointer_type(tl.uint32)) + local_out = tl.load(ptrs_out + RANK).to(tl.pointer_type(tl.uint32)) + value_in = tl.load(local_in + offsets, mask=mask, other=_EMPTY_PAIR) + value_out = tl.load(local_out + offsets, mask=mask, other=_EMPTY_PAIR) + bad = tl.max( + tl.where(mask & ((value_in != _EMPTY_PAIR) | (value_out != _EMPTY_PAIR)), 1, 0) + ) + if bad != 0: + tl.atomic_max(bad_ptr, 1) + + +class LamportRSConv: + """Persistent symmetric buffers for one TP group.""" + + def _initialize_lamport_buffers(self) -> None: + """Arm every slot and collectively verify the exact sentinel bits. + + Do not replace this with a zero-fill: Lamport readiness distinguishes + the bf16 bit pattern for -0.0 (0x8000) from every published payload. + The validation is deliberately collective so one rank cannot enter the + first polling kernel while another rank still has an unarmed buffer. + """ + if not self.buf_in.is_contiguous() or not self.buf_out.is_contiguous(): + raise RuntimeError("Lamport symmetric buffers must be contiguous") + if self.buf_in.numel() % 2 or self.buf_out.numel() % 2: + raise RuntimeError("Lamport buffers must contain whole uint32 pairs") + + # Fill through int16 so each bf16 lane receives the exact -0.0 bits. + # This covers all three generations and all max-token slots, including + # slots that a smaller first invocation does not touch. + self.buf_in.view(torch.int16).fill_(-0x8000) + self.buf_out.view(torch.int16).fill_(-0x8000) + torch.accelerator.synchronize(self.device) + self.tp.barrier() + + # Scan the complete local allocations. Since every rank performs the + # scan and participates in MAX, success proves every symmetric backing + # allocation was armed before any rank is allowed to use generation 0. + bad = torch.logical_or( + self.buf_in.view(torch.int16).ne(-0x8000).any(), + self.buf_out.view(torch.int16).ne(-0x8000).any(), + ).to(dtype=torch.float32) + bad = self.tp.all_reduce(bad) + if int(bad.item()) != 0: + raise RuntimeError("Lamport sentinel initialization failed on a TP rank") + self.tp.barrier() + + def _initialize_mnnvl_buffers(self) -> None: + """Initialize and validate FlashInfer fabric-mapped Lamport storage.""" + self._mnnvl_input_handle.lamport_initialize(self.rank, torch.bfloat16) + self._mnnvl_output_handle.lamport_initialize(self.rank, torch.bfloat16) + torch.accelerator.synchronize(self.device) + self.tp.barrier() + + bad = torch.zeros((), dtype=torch.int32, device=self.device) + num_pairs = self.num_buffers * self.max_tokens * self.hidden_size // 2 + _validate_lamport_init_kernel[(triton.cdiv(num_pairs, 256),)]( + self.input_peer_ptrs, + self.output_peer_ptrs, + bad, + num_pairs, + RANK=self.rank, + BLOCK=256, + num_warps=4, + ) + bad = self.tp.all_reduce(bad.to(torch.float32)) + if int(bad.item()) != 0: + raise RuntimeError("MNNVL Lamport sentinel initialization failed") + self.tp.barrier() + + def __init__( + self, hidden_size: int, window_size: int, max_tokens: int = _MAX_TOKENS + ) -> None: + import torch.distributed._symmetric_memory as symm_mem + + tp = get_tp_group() + self.tp = tp + self.group = tp.device_group + self.world_size = tp.world_size + self.rank = tp.rank_in_group + self.device = torch.device(tp.device) + if self.world_size not in (2, 4, 8): + raise ValueError(f"TP world size must be 2, 4, or 8, got {self.world_size}") + if hidden_size % (2 * self.world_size) != 0: + raise ValueError("hidden size must produce an even shard on every rank") + if window_size != 4: + raise ValueError(f"short-conv window size must be 4, got {window_size}") + if max_tokens < 1 or max_tokens > _MAX_TOKENS: + raise ValueError(f"max_tokens must be in [1, {_MAX_TOKENS}]") + + is_cross_node = not all(in_the_same_node_as(tp.cpu_group)) + self.hidden_size = hidden_size + self.window_size = window_size + self.max_tokens = max_tokens + self.shard_size = hidden_size // self.world_size + # Three generations follow FlashInfer's Lamport layout. A generation + # is reused only after two intervening collective calls. + self.num_buffers = 3 + self.input_generation_bytes = max_tokens * hidden_size * 2 + self.output_generation_bytes = max_tokens * hidden_size * 2 + self.use_pdl = torch.cuda.get_device_capability(self.device)[0] >= 9 + if is_cross_node: + try: + from flashinfer.comm.mnnvl import ( + McastGPUBuffer, + TorchDistBackend, + is_mnnvl_fabric_supported, + ) + except ImportError as error: + raise RuntimeError( + "cross-node TP requires FlashInfer MNNVL support" + ) from error + + local_supported = int( + is_mnnvl_fabric_supported(torch.accelerator.current_device_index()) + ) + unsupported = torch.tensor( + 1 - local_supported, dtype=torch.float32, device=self.device + ) + unsupported = tp.all_reduce(unsupported) + if int(unsupported.item()) != 0: + raise RuntimeError("cross-node TP is supported only on MNNVL fabric") + + comm_backend = TorchDistBackend(self.group) + allocation_bytes = self.num_buffers * max_tokens * hidden_size * 2 + self._mnnvl_input_handle = McastGPUBuffer( + allocation_bytes, + self.world_size, + self.rank, + self.device, + comm_backend, + ) + self._mnnvl_output_handle = McastGPUBuffer( + allocation_bytes, + self.world_size, + self.rank, + self.device, + comm_backend, + ) + self.input_peer_ptrs = self._mnnvl_input_handle.get_buffer_ptrs_dev() + self.output_peer_ptrs = self._mnnvl_output_handle.get_buffer_ptrs_dev() + self._initialize_mnnvl_buffers() + logger.info("using FlashInfer fabric-mapped MNNVL Lamport buffers") + else: + self.buf_in = symm_mem.empty( + self.num_buffers, + max_tokens, + self.world_size, + self.shard_size, + dtype=torch.bfloat16, + device=self.device, + ) + self.buf_out = symm_mem.empty( + self.num_buffers, + max_tokens, + hidden_size, + dtype=torch.bfloat16, + device=self.device, + ) + group_name = self.group.group_name + input_handle = symm_mem.rendezvous(self.buf_in, group_name) + output_handle = symm_mem.rendezvous(self.buf_out, group_name) + self.input_peer_ptrs = input_handle.buffer_ptrs_dev + self.output_peer_ptrs = output_handle.buffer_ptrs_dev + self._input_handle = input_handle + self._output_handle = output_handle + self._initialize_lamport_buffers() + self.generation = 0 + + def usable(self, num_tokens: int) -> bool: + return 0 < num_tokens <= self.max_tokens + + def rs_sconv_ag_add_norm( + self, + input_tensor: torch.Tensor, + residual: torch.Tensor, + conv_weight: torch.Tensor, + norm_weight: torch.Tensor | None, + eps: float, + cache: torch.Tensor, + positions: torch.Tensor, + block_table: torch.Tensor, + seq_idx: torch.Tensor, + slot_mapping: torch.Tensor, + off_s: int, + ws: int, + block_size: int, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + """Return ``(normed | None, new_residual)``, both shaped ``[T, 6144]``.""" + tokens, hidden_size = residual.shape + if not self.usable(tokens): + raise ValueError(f"num_tokens must be in [1, {self.max_tokens}]") + if hidden_size != self.hidden_size or residual.dtype != torch.bfloat16: + raise ValueError("residual must be bf16 [T, 6144]") + if ( + input_tensor.shape != residual.shape + or input_tensor.dtype != torch.bfloat16 + or input_tensor.stride(1) != 1 + ): + raise ValueError("input_tensor must be channel-contiguous bf16 [T, 6144]") + shard_size = hidden_size // self.world_size + if conv_weight.shape != (shard_size, self.window_size): + raise ValueError( + f"conv_weight must have shape [{shard_size}, {self.window_size}]" + ) + if ( + conv_weight.dtype != torch.bfloat16 + or conv_weight.stride(0) != self.window_size + ): + raise ValueError("conv_weight must be contiguous bf16") + if norm_weight is not None and ( + norm_weight.shape != (hidden_size,) or norm_weight.dtype != torch.bfloat16 + ): + raise ValueError("norm_weight must be bf16 [6144] or None") + if cache.dtype != torch.bfloat16 or cache.ndim != 4: + raise ValueError("cache must be a 4-D bf16 tensor") + if shard_size % ws != 0 or cache.shape[1] != shard_size // ws: + raise ValueError("cache head layout is inconsistent with ws") + if off_s < 0 or off_s + ws > cache.shape[3]: + raise ValueError("cache channel offset is out of bounds") + + index = self.generation + input_offset = index * self.input_generation_bytes // 4 + output_offset = index * self.output_generation_bytes // 4 + normed = torch.empty_like(residual) if norm_weight is not None else None + residual_out = torch.empty_like(residual) + phase_splits = 8 if tokens == 1 and shard_size % 8 == 0 else 1 + phase_tile_p2 = triton.next_power_of_2(shard_size // phase_splits) + phase_grid = (tokens, phase_splits) + # Wide CTAs win before the grid saturates; smaller CTAs reduce register + # pressure once high-throughput batches provide enough parallelism. + if 128 <= tokens <= 2048: + phase_warps = 16 + elif tokens > 2048: + phase_warps = 8 + else: + phase_warps = 4 + gather_warps = 4 if tokens >= 256 else 8 + _publish_input_kernel[phase_grid]( + input_tensor, + self.input_peer_ptrs, + input_offset, + input_tensor.stride(0), + C=hidden_size, + CS=shard_size, + CS_P2=phase_tile_p2, + SPLITS=phase_splits, + RANK=self.rank, + WORLD=self.world_size, + USE_PDL=self.use_pdl, + launch_pdl=self.use_pdl, + num_warps=phase_warps, + ) + _reduce_insert_kernel[phase_grid]( + self.input_peer_ptrs, + input_offset, + cache, + slot_mapping, + cache.stride(0), + cache.stride(1), + cache.stride(2), + cache.stride(3), + block_size, + C=hidden_size, + CS=shard_size, + CS_P2=phase_tile_p2, + SPLITS=phase_splits, + HEAD_SIZE=ws, + CACHE_OFFSET=off_s, + RANK=self.rank, + WORLD=self.world_size, + USE_PDL=self.use_pdl, + launch_pdl=self.use_pdl, + num_warps=phase_warps, + ) + _sconv_publish_kernel[phase_grid]( + self.output_peer_ptrs, + output_offset, + residual, + conv_weight, + cache, + positions, + seq_idx, + slot_mapping, + block_table, + residual.stride(0), + cache.stride(0), + cache.stride(1), + cache.stride(2), + cache.stride(3), + block_table.stride(0), + block_table.shape[1], + block_size, + C=hidden_size, + CS=shard_size, + CS_P2=phase_tile_p2, + SPLITS=phase_splits, + HEAD_SIZE=ws, + CACHE_OFFSET=off_s, + RANK=self.rank, + WORLD=self.world_size, + WINDOW=self.window_size, + USE_PDL=self.use_pdl, + launch_pdl=self.use_pdl, + num_warps=phase_warps, + ) + _gather_norm_kernel[(tokens,)]( + self.output_peer_ptrs, + output_offset, + norm_weight if norm_weight is not None else residual, + normed if normed is not None else residual_out, + residual_out, + eps, + residual.stride(0), + C=hidden_size, + C_P2=triton.next_power_of_2(hidden_size), + RANK=self.rank, + HAS_NORM=norm_weight is not None, + USE_PDL=self.use_pdl, + launch_pdl=self.use_pdl, + num_warps=gather_warps, + ) + self.generation = (index + 1) % self.num_buffers + return normed, residual_out + + +_STATE: LamportRSConv | None = None +_STATE_FAILED = False + + +def initialize_lamport_rs_conv( + hidden_size: int, window_size: int, max_num_batched_tokens: int +) -> None: + """Collectively initialize the TP-group state during model construction.""" + global _STATE, _STATE_FAILED + if _STATE is not None: + if _STATE.hidden_size != hidden_size or _STATE.window_size != window_size: + raise RuntimeError("all Lamport users must share hidden and window sizes") + return + if _STATE_FAILED or os.environ.get("LAMPORT_RS_SCONV", "1") == "0": + return + try: + max_tokens = min(_MAX_TOKENS, max_num_batched_tokens) + _STATE = LamportRSConv(hidden_size, window_size, max_tokens=max_tokens) + except Exception: + _STATE_FAILED = True + logger.exception("fused collective unavailable; use the NCCL fallback") + + +def get_lamport_rs_conv(hidden_size: int, window_size: int) -> LamportRSConv | None: + """Return the state initialized with the model, or ``None`` for fallback.""" + if _STATE is not None and ( + _STATE.hidden_size != hidden_size or _STATE.window_size != window_size + ): + raise RuntimeError("all Lamport users must share hidden and window sizes") + return _STATE diff --git a/vllm/models/inkling/amd/ops/mm_towers.py b/vllm/models/inkling/amd/ops/mm_towers.py new file mode 100644 index 0000000000..021cf5274b --- /dev/null +++ b/vllm/models/inkling/amd/ops/mm_towers.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fused CUDA kernels for the Inkling vision/audio towers. + +Both kernels keep the reference paths' fp32 accumulation and per-op bf16 +rounding points (native ``rms_norm`` / ``F.gelu``); outputs are frequently +bit-identical and otherwise differ by 1-2 bf16 ulps from reduction-order +(real-checkpoint-weight cosine vs reference > 0.9999998). +""" + +from __future__ import annotations + +import torch + +from vllm.triton_utils import tl, tldevice, triton + +from .norm import _get_num_warps_from_block_size + + +@triton.jit +def _dmel_embed_sum_norm_kernel( + idx_ptr, # [T, NB] int32 dMel bin indices (values in [0, VOCAB)) + w_ptr, # [NB * VOCAB, D] bf16 embedding table + norm_w_ptr, # [D] (unused if HAS_NORM=False) + out_ptr, # [T, D] bf16 + eps, + D, + stride_idx_t, + NB: tl.constexpr, + VOCAB: tl.constexpr, + D_P2: tl.constexpr, + HAS_NORM: tl.constexpr, +): + t = tl.program_id(0).to(tl.int64) + offs = tl.arange(0, D_P2) + mask = offs < D + # One embedding row per mel bin (bin b uses table rows [b*VOCAB, (b+1)*VOCAB)), + # summed in fp32 (matches torch's fp32-accumulated bf16 .sum()). + acc = tl.zeros([D_P2], dtype=tl.float32) + for b in tl.static_range(NB): + v = tl.load(idx_ptr + t * stride_idx_t + b) + row = (b * VOCAB + v).to(tl.int64) + acc += tl.load(w_ptr + row * D + offs, mask=mask, other=0.0).to(tl.float32) + h = acc.to(tl.bfloat16) + if HAS_NORM: + # Match ir.ops.rms_norm: fp32 variance/normalize, then a single-rounded + # bf16 multiply with the bf16 weight. + x32 = h.to(tl.float32) + var = tl.sum(tl.where(mask, x32 * x32, 0.0), axis=0) / D + xn = (x32 * tl.math.rsqrt(var + eps)).to(tl.bfloat16) + w = tl.load(norm_w_ptr + offs, mask=mask, other=0.0) + h = xn * w + tl.store(out_ptr + t * D + offs, h, mask=mask) + + +def dmel_embed_sum_norm( + dmel_idx: torch.Tensor, # [T, NB] int32 + weight: torch.Tensor, # [NB * VOCAB, D] bf16 + norm_weight: torch.Tensor | None, + eps: float, +) -> torch.Tensor: + """``rmsnorm(sum_b weight[b * VOCAB + idx[:, b]])`` in one launch (no + [T, NB, D] intermediate).""" + T, nb = dmel_idx.shape + D = weight.shape[1] + assert weight.shape[0] % nb == 0 + vocab = weight.shape[0] // nb + out = torch.empty((T, D), dtype=weight.dtype, device=weight.device) + if T == 0: + return out + d_p2 = triton.next_power_of_2(D) + _dmel_embed_sum_norm_kernel[(T,)]( + dmel_idx, + weight, + norm_weight if norm_weight is not None else weight, + out, + eps, + D, + dmel_idx.stride(0), + NB=nb, + VOCAB=vocab, + D_P2=d_p2, + HAS_NORM=norm_weight is not None, + # Swept on GB200: 8 warps beats the block-size heuristic's 16 by ~4% + # at large T (the 80-row gather chain is latency- not lane-bound). + num_warps=8, + ) + return out + + +@triton.jit +def _rmsnorm_gelu_kernel( + x_ptr, # [R, D] bf16 + w_ptr, # [D] + out_ptr, # [R, D] bf16 (or the folded layout when FOLD) + eps, + R, + D, + D_P2: tl.constexpr, + BLOCK_M: tl.constexpr, # rows per block (>1 for small D) + HAS_GELU: tl.constexpr, + FOLD: tl.constexpr, + # fold geometry: input rows index [N, T, H, W]; the store scatters each + # row to (out_row, slot) of fold_timespace_to_depth's output layout. + FT: tl.constexpr, + FH: tl.constexpr, + FW: tl.constexpr, + TF: tl.constexpr, + HF: tl.constexpr, +): + pid = tl.program_id(0).to(tl.int64) + rows = pid * BLOCK_M + tl.arange(0, BLOCK_M) + rmask = rows < R + offs = tl.arange(0, D_P2) + mask = rmask[:, None] & (offs < D)[None, :] + x32 = tl.load(x_ptr + rows[:, None] * D + offs[None, :], mask=mask, other=0.0).to( + tl.float32 + ) + var = tl.sum(x32 * x32, axis=1) / D + xn = (x32 * tl.math.rsqrt(var + eps)[:, None]).to(tl.bfloat16) + w = tl.load(w_ptr + offs, mask=offs < D, other=0.0) + h = xn * w[None, :] # bf16 multiply, matching ir.ops.rms_norm + if HAS_GELU: + # Exact (erf) GELU on the bf16-rounded norm output, fp32 math, + # matching F.gelu's opmath on a bf16 tensor. + g32 = h.to(tl.float32) + h = (0.5 * g32 * (1.0 + tldevice.erf(g32 * 0.7071067811865476))).to(tl.bfloat16) + if FOLD: + # Store directly into the next layer's folded layout (a pure + # permutation — replaces the separate fold copy pass). + t = (rows // (FH * FW)) % FT + hh = (rows // FW) % FH + ww = rows % FW + n = rows // (FT * FH * FW) + slot = ((t % TF) * HF + hh % HF) * HF + ww % HF + out_row = ((n * (FT // TF) + t // TF) * (FH // HF) + hh // HF) * ( + FW // HF + ) + ww // HF + base = (out_row * (TF * HF * HF) + slot) * D + tl.store(out_ptr + base[:, None] + offs[None, :], h, mask=mask) + else: + tl.store(out_ptr + rows[:, None] * D + offs[None, :], h, mask=mask) + + +def rmsnorm_gelu( + x: torch.Tensor, # [..., D] bf16 contiguous + weight: torch.Tensor, + eps: float, + gelu: bool = True, + fold: tuple[int, int] | None = None, # (t_fold, hw_fold) of the NEXT fold +) -> torch.Tensor: + """Fused ``gelu(rmsnorm(x))`` (or plain rmsnorm); multiple rows per block + when D is small. With ``fold``, x must be [N, T, H, W, D] and the output + comes back as ``fold_timespace_to_depth(result, *fold)``.""" + D = x.shape[-1] + flat = x.reshape(-1, D) + assert flat.stride(1) == 1 and flat.stride(0) == D + R = flat.shape[0] + if fold is None: + out = torch.empty_like(flat) + ft = fh = fw = tf = hf = 1 + out_shape = x.shape + else: + tf, hf = fold + N, ft, fh, fw, _ = x.shape + out_shape = (N, ft // tf, fh // hf, fw // hf, tf * hf * hf * D) + out = torch.empty(out_shape, dtype=x.dtype, device=x.device) + if R == 0: + return out.reshape(out_shape) + d_p2 = triton.next_power_of_2(D) + block_m = max(1, 4096 // d_p2) + _rmsnorm_gelu_kernel[(triton.cdiv(R, block_m),)]( + flat, + weight, + out, + eps, + R, + D, + D_P2=d_p2, + BLOCK_M=block_m, + HAS_GELU=gelu, + FOLD=fold is not None, + FT=ft, + FH=fh, + FW=fw, + TF=tf, + HF=hf, + num_warps=_get_num_warps_from_block_size(d_p2 * block_m), + ) + return out.reshape(out_shape) diff --git a/vllm/models/inkling/amd/ops/norm.py b/vllm/models/inkling/amd/ops/norm.py new file mode 100644 index 0000000000..d34c1c3259 --- /dev/null +++ b/vllm/models/inkling/amd/ops/norm.py @@ -0,0 +1,414 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +from functools import lru_cache + +import torch + +from vllm.triton_utils import tl, triton + +_MAX_FUSED_SIZE = 65536 + + +def _get_num_warps_from_block_size(block_size: int) -> int: + if block_size >= 32768: + return 32 + if block_size >= 8192: + return 16 + if block_size >= 2048: + return 8 + return 4 + + +def _largest_power_of_2(n: int) -> int: + assert n > 0, f"{n=}" + return 1 << (n.bit_length() - 1) + + +@lru_cache(maxsize=128) +def _get_grid_size_for_mem_bw_kernel(device: torch.device, factor: int = 8) -> int: + num_sms = torch.cuda.get_device_properties(device).multi_processor_count + return _largest_power_of_2(num_sms) * factor + + +@triton.jit +def _rmsnorm_fwd_kernel( + x_ptr, + weight_ptr, + y_ptr, + rstd_ptr, + eps, + x_stride_0, + y_stride_0, + n_cols, + block_size_n: tl.constexpr, +): + pid_m = tl.program_id(0).to(tl.int64) + offs_n = tl.arange(0, block_size_n) + mask_n = offs_n < n_cols + + weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + x = tl.load(x_ptr + pid_m * x_stride_0 + offs_n, mask=mask_n, other=0.0).to( + tl.float32 + ) + row_var = tl.sum(x * x, axis=0) / n_cols + rstd = tl.math.rsqrt(row_var + eps) + tl.store(rstd_ptr + pid_m, rstd) + + y = x * rstd * weight + tl.store(y_ptr + pid_m * y_stride_0 + offs_n, y, mask=mask_n) + + +@triton.jit(do_not_specialize=["n_rows"]) +def _rmsnorm_fwd_kernel_block_m( + x_ptr, + weight_ptr, + y_ptr, + rstd_ptr, + eps, + x_stride_0, + y_stride_0, + n_rows, + n_cols, + block_size_m: tl.constexpr, + block_size_n: tl.constexpr, +): + pid_m = tl.program_id(0).to(tl.int64) + offs_n = tl.arange(0, block_size_n) + mask_n = offs_n < n_cols + + num_blocks_m = tl.cdiv(n_rows, block_size_m) + blocks_per_pid = tl.cdiv(num_blocks_m, tl.num_programs(0)) + block_id_start = pid_m * blocks_per_pid + block_id_end = min(block_id_start + blocks_per_pid, num_blocks_m) + + weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + + for block_id in range(block_id_start, block_id_end): + offs_m = block_id * block_size_m + tl.arange(0, block_size_m) + mask_m = offs_m < n_rows + mask_mn = mask_m[:, None] & mask_n[None, :] + x = tl.load( + x_ptr + offs_m[:, None] * x_stride_0 + offs_n[None, :], + mask=mask_mn, + other=0.0, + ).to(tl.float32) + row_var = tl.sum(x * x, axis=1) / n_cols + rstd = tl.math.rsqrt(row_var + eps) + tl.store(rstd_ptr + offs_m, rstd, mask=mask_m) + + y = x * rstd[:, None] * weight + tl.store( + y_ptr + offs_m[:, None] * y_stride_0 + offs_n[None, :], + y, + mask=mask_mn, + ) + + +@triton.jit +def _add_rmsnorm_fwd_kernel( + res_ptr, # [T, N] residual (read) + delta_ptr, # [T, N] delta to add (read) + weight_ptr, + y_ptr, # [T, N] normed output + res_out_ptr, # [T, N] updated residual output + eps, + res_stride_0, + delta_stride_0, + y_stride_0, + res_out_stride_0, + n_cols, + block_size_n: tl.constexpr, +): + pid_m = tl.program_id(0).to(tl.int64) + offs_n = tl.arange(0, block_size_n) + mask_n = offs_n < n_cols + + weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + r = tl.load(res_ptr + pid_m * res_stride_0 + offs_n, mask=mask_n, other=0.0).to( + tl.float32 + ) + d = tl.load(delta_ptr + pid_m * delta_stride_0 + offs_n, mask=mask_n, other=0.0).to( + tl.float32 + ) + # Round the sum to the residual dtype first (matches the eager + # `residual + delta` then rmsnorm-on-bf16 sequence bit-for-bit). + s = (r + d).to(res_out_ptr.dtype.element_ty) + tl.store(res_out_ptr + pid_m * res_out_stride_0 + offs_n, s, mask=mask_n) + x = s.to(tl.float32) + row_var = tl.sum(x * x, axis=0) / n_cols + rstd = tl.math.rsqrt(row_var + eps) + y = x * rstd * weight + tl.store(y_ptr + pid_m * y_stride_0 + offs_n, y, mask=mask_n) + + +def add_rmsnorm( + residual: torch.Tensor, + delta: torch.Tensor, + weight: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + """Fused ``res = residual + delta; y = rmsnorm(res)``. + + Returns ``(y, res)``; both are fresh tensors (cudagraph-friendly, no + in-place update of the inputs). + """ + assert residual.ndim == 2 and delta.ndim == 2, (residual.shape, delta.shape) + n_rows, n_cols = residual.shape + assert weight.shape[0] == n_cols + y = torch.empty_like(residual) + res_out = torch.empty_like(residual) + if n_rows == 0: + return y, res_out + + block_size_n = triton.next_power_of_2(n_cols) + max_block_size_n = _MAX_FUSED_SIZE // residual.element_size() + if max_block_size_n < block_size_n: + raise RuntimeError(f"Large {n_cols=} is not supported") + num_warps = _get_num_warps_from_block_size(block_size_n) + _add_rmsnorm_fwd_kernel[(n_rows,)]( + residual, + delta, + weight, + y, + res_out, + eps, + residual.stride(0), + delta.stride(0), + y.stride(0), + res_out.stride(0), + n_cols, + block_size_n, + num_warps=num_warps, + ) + return y, res_out + + +@triton.jit +def _embed_rmsnorm_kernel( + ids_ptr, # [T] token ids + table_ptr, # [V, N] embedding table + weight_ptr, # [N] (HAS_NORM only) + chain_weight_ptr, # [N] (HAS_CHAIN only) + out_ptr, # [T, N] rmsnorm(table[ids], weight) + chain_out_ptr, # [T, N] rmsnorm(out, chain_weight) (HAS_CHAIN only) + eps, + table_stride_0, + n_cols, + block_size_n: tl.constexpr, + HAS_NORM: tl.constexpr, + HAS_CHAIN: tl.constexpr, +): + pid_m = tl.program_id(0).to(tl.int64) + offs_n = tl.arange(0, block_size_n) + mask_n = offs_n < n_cols + row = tl.load(ids_ptr + pid_m).to(tl.int64) + x = tl.load(table_ptr + row * table_stride_0 + offs_n, mask=mask_n, other=0.0) + if HAS_NORM: + xf = x.to(tl.float32) + rstd = tl.math.rsqrt(tl.sum(xf * xf, axis=0) / n_cols + eps) + w = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + # Round to the output dtype so the chained norm is bit-exact vs the + # unfused pair (which stores bf16 in between). + x = (xf * rstd * w).to(out_ptr.dtype.element_ty) + tl.store(out_ptr + pid_m * n_cols + offs_n, x, mask=mask_n) + if HAS_CHAIN: + xf = x.to(tl.float32) + rstd = tl.math.rsqrt(tl.sum(xf * xf, axis=0) / n_cols + eps) + w = tl.load(chain_weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + tl.store( + chain_out_ptr + pid_m * n_cols + offs_n, + (xf * rstd * w).to(chain_out_ptr.dtype.element_ty), + mask=mask_n, + ) + + +def embed_rmsnorm( + input_ids: torch.Tensor, + embed_table: torch.Tensor, + weight: torch.Tensor | None, + eps: float, + chain_weight: torch.Tensor | None = None, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """Fused ``rmsnorm(embed_table[input_ids], weight)`` row gather + norm. + + Requires the full vocab on-rank (replicated or tp_size == 1). + ``weight=None`` skips the norm (``use_embed_norm=False``), leaving a pure + embedding-table gather. + ``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the + first decoder layer's pre-attention norm) as a second output, still one + launch. Bit-exact vs the unfused module sequence.""" + ids = input_ids.view(-1) + (T,) = ids.shape + n = embed_table.shape[1] + out = torch.empty( + (*input_ids.shape, n), dtype=embed_table.dtype, device=embed_table.device + ) + chain_out = torch.empty_like(out) if chain_weight is not None else None + if T > 0: + block_size_n = triton.next_power_of_2(n) + _embed_rmsnorm_kernel[(T,)]( + ids, + embed_table, + weight if weight is not None else embed_table, + chain_weight if chain_weight is not None else embed_table, + out, + chain_out if chain_out is not None else out, + eps, + embed_table.stride(0), + n, + block_size_n, + HAS_NORM=weight is not None, + HAS_CHAIN=chain_weight is not None, + num_warps=_get_num_warps_from_block_size(block_size_n), + ) + if chain_out is not None: + return out, chain_out + return out + + +@triton.jit +def _embed_dual_rmsnorm_cat_kernel( + hidden_ptr, # [T, N] + emb_ptr, # [T, N] embeddings, or the [V, N] embedding table when GATHER + ids_ptr, # [T] token ids (GATHER only) + w_hidden_ptr, # [N] + w_pre_ptr, # [N] chained pre-norm on the embed side (HAS_PRE_NORM only) + w_embed_ptr, # [N] + out_ptr, # [T, 2N]: [rmsnorm(hidden) | rmsnorm(rmsnorm?(emb))] + eps, + hidden_stride_0, + emb_stride_0, + n_cols, + block_size_n: tl.constexpr, + GATHER: tl.constexpr, + HAS_PRE_NORM: tl.constexpr, +): + pid_m = tl.program_id(0).to(tl.int64) + which = tl.program_id(1) # 0 -> hidden into cols [0, N); 1 -> emb into [N, 2N) + offs_n = tl.arange(0, block_size_n) + mask_n = offs_n < n_cols + if which == 0: + x = tl.load( + hidden_ptr + pid_m * hidden_stride_0 + offs_n, mask=mask_n, other=0.0 + ).to(tl.float32) + w = tl.load(w_hidden_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + else: + row = tl.load(ids_ptr + pid_m).to(tl.int64) if GATHER else pid_m + x = tl.load(emb_ptr + row * emb_stride_0 + offs_n, mask=mask_n, other=0.0).to( + tl.float32 + ) + if HAS_PRE_NORM: + w_pre = tl.load(w_pre_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / n_cols + eps) + # Round-trip through the output dtype so the chained norm is + # bit-exact vs the unfused pair (which stores bf16 in between). + x = (x * rstd * w_pre).to(out_ptr.dtype.element_ty).to(tl.float32) + w = tl.load(w_embed_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / n_cols + eps) + tl.store( + out_ptr + pid_m * (2 * n_cols) + which * n_cols + offs_n, + (x * rstd * w).to(out_ptr.dtype.element_ty), + mask=mask_n, + ) + + +def embed_dual_rmsnorm_cat( + hidden: torch.Tensor, + hidden_weight: torch.Tensor, + embed_weight: torch.Tensor, + eps: float, + *, + embeds: torch.Tensor | None = None, + input_ids: torch.Tensor | None = None, + embed_table: torch.Tensor | None = None, + pre_norm_weight: torch.Tensor | None = None, +) -> torch.Tensor: + """The MTP depth-layer input in one launch: + ``cat([rmsnorm(hidden, w_h), rmsnorm(pre?(emb), w_e)], -1)``. + + The embed side is either a fused row gather ``embed_table[input_ids]`` + (draft decode steps) or precomputed ``embeds`` ([T, N], the target-merged + multimodal embeddings at draft prefill); ``pre_norm_weight`` chains the + backbone embed_norm in front of the depth embed_norm (bit-exact vs the + unfused sequence). The concat copies collapse into direct writes.""" + T, n = hidden.shape + if embeds is not None: + assert embeds.shape == hidden.shape + src, ids, src_stride = embeds, embeds, embeds.stride(0) + gather = False + else: + assert input_ids is not None and embed_table is not None + assert input_ids.shape == (T,) and embed_table.shape[1] == n + src, ids, src_stride = embed_table, input_ids, embed_table.stride(0) + gather = True + out = torch.empty((T, 2 * n), dtype=hidden.dtype, device=hidden.device) + if T == 0: + return out + block_size_n = triton.next_power_of_2(n) + _embed_dual_rmsnorm_cat_kernel[(T, 2)]( + hidden, + src, + ids, + hidden_weight, + pre_norm_weight if pre_norm_weight is not None else embed_weight, + embed_weight, + out, + eps, + hidden.stride(0), + src_stride, + n, + block_size_n, + GATHER=gather, + HAS_PRE_NORM=pre_norm_weight is not None, + num_warps=_get_num_warps_from_block_size(block_size_n), + ) + return out + + +def rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: + assert x.ndim == 2, f"{x.shape=}" + assert weight.ndim == 1, f"{weight.shape=}" + n_rows, n_cols = x.shape + assert weight.shape[0] == n_cols, f"{weight.shape=} {x.shape=}" + y = torch.empty_like(x) + rstd = torch.empty((n_rows,), dtype=torch.float32, device=x.device) + + block_size_n = triton.next_power_of_2(n_cols) + max_block_size_n = _MAX_FUSED_SIZE // x.element_size() + if max_block_size_n < block_size_n: + raise RuntimeError(f"Large {n_cols=} is not supported") + block_size_m = max(1, 4096 // block_size_n) + num_warps = _get_num_warps_from_block_size(block_size_n) + + if block_size_m == 1: + _rmsnorm_fwd_kernel[(n_rows,)]( + x, + weight, + y, + rstd, + eps, + x.stride(0), + y.stride(0), + n_cols, + block_size_n, + num_warps=num_warps, + ) + else: + grid_size = _get_grid_size_for_mem_bw_kernel(x.device) + _rmsnorm_fwd_kernel_block_m[(grid_size,)]( + x, + weight, + y, + rstd, + eps, + x.stride(0), + y.stride(0), + n_rows, + n_cols, + block_size_m, + block_size_n, + num_warps=num_warps, + ) + return y diff --git a/vllm/models/inkling/amd/ops/qkvr_prep.py b/vllm/models/inkling/amd/ops/qkvr_prep.py new file mode 100644 index 0000000000..a910e72170 --- /dev/null +++ b/vllm/models/inkling/amd/ops/qkvr_prep.py @@ -0,0 +1,918 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import aux_stream + +LOW_BLOCK_M = 32 +LOW_BLOCK_N = 64 +LOW_NUM_WARPS = 4 +THROUGHPUT_BLOCK_M = 32 +THROUGHPUT_BLOCK_N = 128 +THROUGHPUT_GROUP_M = 2 +THROUGHPUT_NUM_WARPS = 4 +SMALL_TOKEN_THRESHOLD = 128 +SMALL_NUM_WARPS = 2 +Q_BLOCK_ROWS = 8 +Q_NUM_WARPS = 2 +KV_BLOCK_ROWS = 4 +KV_NUM_WARPS = 2 + + +@triton.jit(do_not_specialize=["rows"]) +def _rel_proj_low_latency_kernel( + qkvr_ptr, + rel_proj_ptr, + rel_out_ptr, + log_scaling_ptr, + rows, + stride_x_t, + R_OFFSET: tl.constexpr, + NUM_Q_HEADS: tl.constexpr, + REL_EXTENT: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + APPLY_LOG_SCALING: tl.constexpr, +): + row = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) + col = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + inner = tl.arange(0, 16) + token = row // NUM_Q_HEADS + head = row % NUM_Q_HEADS + relative = tl.load( + qkvr_ptr + + token[:, None] * stride_x_t + + R_OFFSET + + head[:, None] * 16 + + inner[None, :], + mask=row[:, None] < rows, + other=0.0, + ) + projection = tl.load( + rel_proj_ptr + inner[:, None] * REL_EXTENT + col[None, :], + mask=col[None, :] < REL_EXTENT, + other=0.0, + ) + values = tl.dot(relative, projection, out_dtype=tl.float32).to( + rel_out_ptr.dtype.element_ty + ) + values = values.to(tl.float32) + if APPLY_LOG_SCALING: + values *= tl.load(log_scaling_ptr + token, mask=row < rows, other=1.0)[:, None] + tl.store( + rel_out_ptr + row[:, None] * REL_EXTENT + col[None, :], + values.to(rel_out_ptr.dtype.element_ty), + mask=(row[:, None] < rows) & (col[None, :] < REL_EXTENT), + ) + + +@triton.jit(do_not_specialize=["rows"]) +def _rel_proj_throughput_kernel( + qkvr_ptr, + rel_proj_ptr, + rel_out_ptr, + log_scaling_ptr, + rows, + stride_x_t, + R_OFFSET: tl.constexpr, + NUM_Q_HEADS: tl.constexpr, + REL_EXTENT: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + GROUP_M: tl.constexpr, + APPLY_LOG_SCALING: tl.constexpr, +): + row_group = tl.program_id(0) + col = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + inner = tl.arange(0, 16) + projection = tl.load( + rel_proj_ptr + inner[:, None] * REL_EXTENT + col[None, :], + mask=col[None, :] < REL_EXTENT, + other=0.0, + ) + row_offsets = tl.arange(0, BLOCK_M) + for group_offset in tl.static_range(GROUP_M): + row = (row_group * GROUP_M + group_offset) * BLOCK_M + row_offsets + token = row // NUM_Q_HEADS + head = row % NUM_Q_HEADS + relative = tl.load( + qkvr_ptr + + token[:, None] * stride_x_t + + R_OFFSET + + head[:, None] * 16 + + inner[None, :], + mask=row[:, None] < rows, + other=0.0, + ) + values = tl.dot(relative, projection, out_dtype=tl.float32).to( + rel_out_ptr.dtype.element_ty + ) + values = values.to(tl.float32) + if APPLY_LOG_SCALING: + values *= tl.load(log_scaling_ptr + token, mask=row < rows, other=1.0)[ + :, None + ] + tl.store( + rel_out_ptr + row[:, None] * REL_EXTENT + col[None, :], + values.to(rel_out_ptr.dtype.element_ty), + mask=(row[:, None] < rows) & (col[None, :] < REL_EXTENT), + ) + + +def use_rel_proj_throughput(rows: int, rel_extent: int) -> bool: + min_rows = 8192 if rel_extent == 512 else 2048 + return rows >= min_rows + + +def qkvr_rel_proj( + qkvr: torch.Tensor, + rel_proj: torch.Tensor, + rel_out: torch.Tensor, + log_scaling: torch.Tensor | None, + *, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + d_rel: int, +) -> None: + rows = qkvr.shape[0] * num_q_heads + rel_extent = rel_proj.shape[1] + assert d_rel == 16 and rel_proj.shape[0] == 16 + r_offset = num_q_heads * head_dim + 2 * num_kv_heads * head_dim + log_scaling_ptr = log_scaling if log_scaling is not None else qkvr + common = dict( + R_OFFSET=r_offset, + NUM_Q_HEADS=num_q_heads, + REL_EXTENT=rel_extent, + APPLY_LOG_SCALING=log_scaling is not None, + ) + + if use_rel_proj_throughput(rows, rel_extent): + grid = ( + triton.cdiv(rows, THROUGHPUT_BLOCK_M * THROUGHPUT_GROUP_M), + triton.cdiv(rel_extent, THROUGHPUT_BLOCK_N), + ) + _rel_proj_throughput_kernel[grid]( + qkvr, + rel_proj, + rel_out, + log_scaling_ptr, + rows, + qkvr.stride(0), + BLOCK_M=THROUGHPUT_BLOCK_M, + BLOCK_N=THROUGHPUT_BLOCK_N, + GROUP_M=THROUGHPUT_GROUP_M, + num_warps=THROUGHPUT_NUM_WARPS, + **common, + ) + return + + grid = ( + triton.cdiv(rows, LOW_BLOCK_M), + triton.cdiv(rel_extent, LOW_BLOCK_N), + ) + _rel_proj_low_latency_kernel[grid]( + qkvr, + rel_proj, + rel_out, + log_scaling_ptr, + rows, + qkvr.stride(0), + BLOCK_M=LOW_BLOCK_M, + BLOCK_N=LOW_BLOCK_N, + num_warps=LOW_NUM_WARPS, + **common, + ) + + +@triton.jit(do_not_specialize=["tokens", "stride_block_table_req", "max_blocks"]) +def _qkvr_qkv_kernel( + qkvr_ptr, + q_norm_weight_ptr, + q_out_ptr, + rel_proj_ptr, + rel_out_ptr, + k_weight_ptr, + v_weight_ptr, + k_norm_weight_ptr, + conv_cache_ptr, + key_cache_ptr, + value_cache_ptr, + positions_ptr, + seq_idx_ptr, + conv_slot_mapping_ptr, + conv_block_table_ptr, + query_start_ptr, + attention_slot_mapping_ptr, + log_scaling_ptr, + tokens, + eps, + stride_x_t, + stride_cc_block, + stride_cc_head, + stride_cc_token, + stride_cc_dim, + stride_kc_block, + stride_kc_token, + stride_kc_head, + stride_vc_block, + stride_vc_token, + stride_vc_head, + stride_block_table_req, + max_blocks, + conv_block_size, + attention_page_size, + Q_WIDTH: tl.constexpr, + KV_WIDTH: tl.constexpr, + NUM_Q_HEADS: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + WINDOW_SIZE: tl.constexpr, + OFF_K: tl.constexpr, + OFF_V: tl.constexpr, + APPLY_LOG_SCALING: tl.constexpr, + D_REL: tl.constexpr, + REL_EXTENT: tl.constexpr, + REL_EXTENT_PADDED: tl.constexpr, +): + block = tl.program_id(0) + num_q_rows = tokens * NUM_Q_HEADS + dims = tl.arange(0, HEAD_DIM) + + if block < num_q_rows: + row = block + token = row // NUM_Q_HEADS + head = row % NUM_Q_HEADS + values = tl.load( + qkvr_ptr + token * stride_x_t + head * HEAD_DIM + dims, + ).to(tl.float32) + weight = tl.load(q_norm_weight_ptr + dims).to(tl.float32) + rstd = tl.rsqrt(tl.sum(values * values, axis=0) / HEAD_DIM + eps) + normalized = values * rstd * weight + if APPLY_LOG_SCALING: + normalized = normalized.to(q_out_ptr.dtype.element_ty).to(tl.float32) + normalized *= tl.load(log_scaling_ptr + token) + tl.store( + q_out_ptr + row * HEAD_DIM + dims, + normalized.to(q_out_ptr.dtype.element_ty), + ) + rel_cols = tl.arange(0, REL_EXTENT_PADDED) + rel_mask = rel_cols < REL_EXTENT + projected = tl.zeros([REL_EXTENT_PADDED], dtype=tl.float32) + rel_offset = Q_WIDTH + 2 * KV_WIDTH + head * D_REL + for rel_dim in tl.static_range(D_REL): + rel_value = tl.load( + qkvr_ptr + token * stride_x_t + rel_offset + rel_dim + ).to(tl.float32) + proj = tl.load( + rel_proj_ptr + rel_dim * REL_EXTENT + rel_cols, + mask=rel_mask, + other=0.0, + ).to(tl.float32) + projected += rel_value * proj + projected = projected.to(rel_out_ptr.dtype.element_ty).to(tl.float32) + if APPLY_LOG_SCALING: + projected *= tl.load(log_scaling_ptr + token) + tl.store( + rel_out_ptr + row * REL_EXTENT + rel_cols, + projected.to(rel_out_ptr.dtype.element_ty), + mask=rel_mask, + ) + else: + row = block - num_q_rows + if row < tokens * NUM_KV_HEADS: + token = row // NUM_KV_HEADS + head = row % NUM_KV_HEADS + position = tl.load(positions_ptr + token) + request = tl.load(seq_idx_ptr + token) + conv_slot = tl.load(conv_slot_mapping_ptr + token) + query_start = tl.load(query_start_ptr + token) + attention_slot = tl.load(attention_slot_mapping_ptr + token) + valid = conv_slot >= 0 + + k_col = Q_WIDTH + head * HEAD_DIM + v_col = Q_WIDTH + KV_WIDTH + head * HEAD_DIM + k_value = tl.load(qkvr_ptr + token * stride_x_t + k_col + dims) + v_value = tl.load(qkvr_ptr + token * stride_x_t + v_col + dims) + + safe_slot = tl.maximum(conv_slot, 0) + cache_base = ( + conv_cache_ptr + + (safe_slot // conv_block_size) * stride_cc_block + + head * stride_cc_head + + (safe_slot % conv_block_size) * stride_cc_token + ) + tl.store( + cache_base + (OFF_K + dims) * stride_cc_dim, + k_value, + mask=valid, + ) + tl.store( + cache_base + (OFF_V + dims) * stride_cc_dim, + v_value, + mask=valid, + ) + + acc_k = tl.zeros([HEAD_DIM], dtype=tl.float32) + acc_v = tl.zeros([HEAD_DIM], dtype=tl.float32) + for tap in tl.static_range(WINDOW_SIZE): + source_position = position - (WINDOW_SIZE - 1) + tap + source_row = token - (WINDOW_SIZE - 1) + tap + in_window = valid & (source_position >= 0) + intra = in_window & (source_row >= query_start) + cached = in_window & (source_row < query_start) + safe_row = tl.maximum(source_row, 0) + source_k = tl.load( + qkvr_ptr + safe_row * stride_x_t + k_col + dims, + mask=intra, + other=0.0, + ).to(tl.float32) + source_v = tl.load( + qkvr_ptr + safe_row * stride_x_t + v_col + dims, + mask=intra, + other=0.0, + ).to(tl.float32) + safe_position = tl.maximum(source_position, 0) + logical_block = tl.minimum( + safe_position // conv_block_size, max_blocks - 1 + ) + physical_block = tl.load( + conv_block_table_ptr + + request * stride_block_table_req + + logical_block, + mask=cached, + other=0, + ).to(tl.int64) + tap_base = ( + conv_cache_ptr + + physical_block * stride_cc_block + + head * stride_cc_head + + (safe_position % conv_block_size) * stride_cc_token + ) + source_k += tl.load( + tap_base + (OFF_K + dims) * stride_cc_dim, + mask=cached, + other=0.0, + ).to(tl.float32) + source_v += tl.load( + tap_base + (OFF_V + dims) * stride_cc_dim, + mask=cached, + other=0.0, + ).to(tl.float32) + k_weight = tl.load( + k_weight_ptr + (head * HEAD_DIM + dims) * WINDOW_SIZE + tap + ).to(tl.float32) + v_weight = tl.load( + v_weight_ptr + (head * HEAD_DIM + dims) * WINDOW_SIZE + tap + ).to(tl.float32) + acc_k += source_k * k_weight + acc_v += source_v * v_weight + + k_rounded = (acc_k + k_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty) + v_rounded = (acc_v + v_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty) + k_float = k_rounded.to(tl.float32) + k_norm_weight = tl.load(k_norm_weight_ptr + dims).to(tl.float32) + rstd = tl.rsqrt(tl.sum(k_float * k_float, axis=0) / HEAD_DIM + eps) + k_normalized = (k_float * rstd * k_norm_weight).to( + qkvr_ptr.dtype.element_ty + ) + + safe_attention_slot = tl.maximum(attention_slot, 0) + attention_block = safe_attention_slot // attention_page_size + attention_offset = safe_attention_slot % attention_page_size + tl.store( + key_cache_ptr + + attention_block * stride_kc_block + + attention_offset * stride_kc_token + + head * stride_kc_head + + dims, + k_normalized, + mask=attention_slot >= 0, + ) + tl.store( + value_cache_ptr + + attention_block * stride_vc_block + + attention_offset * stride_vc_token + + head * stride_vc_head + + dims, + v_rounded, + mask=attention_slot >= 0, + ) + + +@triton.jit(do_not_specialize=["num_rows"]) +def _q_kernel( + qkvr_ptr, + q_norm_weight_ptr, + q_out_ptr, + log_scaling_ptr, + num_rows, + stride_x_t, + eps, + NUM_Q_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_ROWS: tl.constexpr, + APPLY_LOG_SCALING: tl.constexpr, +): + rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS) + dims = tl.arange(0, HEAD_DIM) + row_mask = rows < num_rows + tokens = rows // NUM_Q_HEADS + heads = rows % NUM_Q_HEADS + values = tl.load( + qkvr_ptr + + tokens[:, None] * stride_x_t + + heads[:, None] * HEAD_DIM + + dims[None, :], + mask=row_mask[:, None], + other=0.0, + ).to(tl.float32) + weight = tl.load(q_norm_weight_ptr + dims).to(tl.float32) + rstd = tl.rsqrt(tl.sum(values * values, axis=1) / HEAD_DIM + eps) + normalized = values * rstd[:, None] * weight[None, :] + if APPLY_LOG_SCALING: + normalized = normalized.to(q_out_ptr.dtype.element_ty).to(tl.float32) + tau = tl.load(log_scaling_ptr + tokens, mask=row_mask, other=1.0) + normalized *= tau[:, None] + tl.store( + q_out_ptr + rows[:, None] * HEAD_DIM + dims[None, :], + normalized.to(q_out_ptr.dtype.element_ty), + mask=row_mask[:, None], + ) + + +@triton.jit(do_not_specialize=["tokens", "stride_block_table_req", "max_blocks"]) +def _kv_kernel( + qkvr_ptr, + k_weight_ptr, + v_weight_ptr, + k_norm_weight_ptr, + conv_cache_ptr, + key_cache_ptr, + value_cache_ptr, + positions_ptr, + seq_idx_ptr, + conv_slot_mapping_ptr, + conv_block_table_ptr, + query_start_ptr, + attention_slot_mapping_ptr, + tokens, + eps, + stride_x_t, + stride_cc_block, + stride_cc_head, + stride_cc_token, + stride_cc_dim, + stride_kc_block, + stride_kc_token, + stride_kc_head, + stride_vc_block, + stride_vc_token, + stride_vc_head, + stride_block_table_req, + max_blocks, + conv_block_size, + attention_page_size, + Q_WIDTH: tl.constexpr, + KV_WIDTH: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + WINDOW_SIZE: tl.constexpr, + OFF_K: tl.constexpr, + OFF_V: tl.constexpr, + BLOCK_ROWS: tl.constexpr, +): + token = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS) + dims = tl.arange(0, HEAD_DIM) + row_mask = token < tokens + head_id = tl.program_id(1) + head = tl.full([BLOCK_ROWS], head_id, tl.int64) + position = tl.load(positions_ptr + token, mask=row_mask, other=0) + request = tl.load(seq_idx_ptr + token, mask=row_mask, other=0) + conv_slot = tl.load(conv_slot_mapping_ptr + token, mask=row_mask, other=-1) + query_start = tl.load(query_start_ptr + token, mask=row_mask, other=0) + attention_slot = tl.load( + attention_slot_mapping_ptr + token, mask=row_mask, other=-1 + ) + valid = row_mask & (conv_slot >= 0) + + k_col = Q_WIDTH + head * HEAD_DIM + v_col = Q_WIDTH + KV_WIDTH + head * HEAD_DIM + k_value = tl.load( + qkvr_ptr + token[:, None] * stride_x_t + k_col[:, None] + dims[None, :], + mask=row_mask[:, None], + other=0.0, + ) + v_value = tl.load( + qkvr_ptr + token[:, None] * stride_x_t + v_col[:, None] + dims[None, :], + mask=row_mask[:, None], + other=0.0, + ) + safe_slot = tl.maximum(conv_slot, 0) + cache_base = ( + conv_cache_ptr + + (safe_slot // conv_block_size) * stride_cc_block + + head * stride_cc_head + + (safe_slot % conv_block_size) * stride_cc_token + ) + tl.store( + cache_base[:, None] + (OFF_K + dims[None, :]) * stride_cc_dim, + k_value, + mask=valid[:, None], + ) + tl.store( + cache_base[:, None] + (OFF_V + dims[None, :]) * stride_cc_dim, + v_value, + mask=valid[:, None], + ) + + acc_k = tl.zeros([BLOCK_ROWS, HEAD_DIM], dtype=tl.float32) + acc_v = tl.zeros([BLOCK_ROWS, HEAD_DIM], dtype=tl.float32) + for tap in tl.static_range(WINDOW_SIZE): + source_position = position - (WINDOW_SIZE - 1) + tap + source_row = token - (WINDOW_SIZE - 1) + tap + in_window = valid & (source_position >= 0) + intra = in_window & (source_row >= query_start) + cached = in_window & (source_row < query_start) + safe_row = tl.maximum(source_row, 0) + source_k = tl.load( + qkvr_ptr + safe_row[:, None] * stride_x_t + k_col[:, None] + dims[None, :], + mask=intra[:, None], + other=0.0, + ).to(tl.float32) + source_v = tl.load( + qkvr_ptr + safe_row[:, None] * stride_x_t + v_col[:, None] + dims[None, :], + mask=intra[:, None], + other=0.0, + ).to(tl.float32) + safe_position = tl.maximum(source_position, 0) + logical_block = tl.minimum(safe_position // conv_block_size, max_blocks - 1) + physical_block = tl.load( + conv_block_table_ptr + request * stride_block_table_req + logical_block, + mask=cached, + other=0, + ).to(tl.int64) + tap_base = ( + conv_cache_ptr + + physical_block * stride_cc_block + + head * stride_cc_head + + (safe_position % conv_block_size) * stride_cc_token + ) + source_k += tl.load( + tap_base[:, None] + (OFF_K + dims[None, :]) * stride_cc_dim, + mask=cached[:, None], + other=0.0, + ).to(tl.float32) + source_v += tl.load( + tap_base[:, None] + (OFF_V + dims[None, :]) * stride_cc_dim, + mask=cached[:, None], + other=0.0, + ).to(tl.float32) + k_weight = tl.load( + k_weight_ptr + (head_id * HEAD_DIM + dims) * WINDOW_SIZE + tap + ).to(tl.float32) + v_weight = tl.load( + v_weight_ptr + (head_id * HEAD_DIM + dims) * WINDOW_SIZE + tap + ).to(tl.float32) + acc_k += source_k * k_weight[None, :] + acc_v += source_v * v_weight[None, :] + + k_rounded = (acc_k + k_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty) + v_rounded = (acc_v + v_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty) + k_float = k_rounded.to(tl.float32) + k_norm_weight = tl.load(k_norm_weight_ptr + dims).to(tl.float32) + rstd = tl.rsqrt(tl.sum(k_float * k_float, axis=1) / HEAD_DIM + eps) + k_normalized = (k_float * rstd[:, None] * k_norm_weight[None, :]).to( + qkvr_ptr.dtype.element_ty + ) + + safe_attention_slot = tl.maximum(attention_slot, 0) + attention_block = safe_attention_slot // attention_page_size + attention_offset = safe_attention_slot % attention_page_size + attention_mask = row_mask & (attention_slot >= 0) + tl.store( + key_cache_ptr + + attention_block[:, None] * stride_kc_block + + attention_offset[:, None] * stride_kc_token + + head[:, None] * stride_kc_head + + dims[None, :], + k_normalized, + mask=attention_mask[:, None], + ) + tl.store( + value_cache_ptr + + attention_block[:, None] * stride_vc_block + + attention_offset[:, None] * stride_vc_token + + head[:, None] * stride_vc_head + + dims[None, :], + v_rounded, + mask=attention_mask[:, None], + ) + + +def _run_tiled_q( + qkvr: torch.Tensor, + q_norm_weight: torch.Tensor, + q_out: torch.Tensor, + positions: torch.Tensor, + *, + eps: float, + num_q_heads: int, + head_dim: int, + log_scaling: torch.Tensor | None, +) -> None: + num_rows = qkvr.shape[0] * num_q_heads + _q_kernel[(triton.cdiv(num_rows, Q_BLOCK_ROWS),)]( + qkvr, + q_norm_weight, + q_out, + log_scaling if log_scaling is not None else positions, + num_rows, + qkvr.stride(0), + eps, + NUM_Q_HEADS=num_q_heads, + HEAD_DIM=head_dim, + BLOCK_ROWS=Q_BLOCK_ROWS, + APPLY_LOG_SCALING=log_scaling is not None, + num_warps=Q_NUM_WARPS, + ) + + +def _run_tiled_kv( + qkvr: torch.Tensor, + k_weight: torch.Tensor, + v_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + conv_cache: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + positions: torch.Tensor, + seq_idx: torch.Tensor, + conv_slot_mapping: torch.Tensor, + conv_block_table: torch.Tensor, + query_start: torch.Tensor, + attention_slot_mapping: torch.Tensor, + *, + eps: float, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + off_k: int, + off_v: int, + conv_block_size: int, +) -> None: + tokens = qkvr.shape[0] + _kv_kernel[(triton.cdiv(tokens, KV_BLOCK_ROWS), num_kv_heads)]( + qkvr, + k_weight, + v_weight, + k_norm_weight, + conv_cache, + key_cache, + value_cache, + positions, + seq_idx, + conv_slot_mapping, + conv_block_table, + query_start, + attention_slot_mapping, + tokens, + eps, + qkvr.stride(0), + conv_cache.stride(0), + conv_cache.stride(1), + conv_cache.stride(2), + conv_cache.stride(3), + key_cache.stride(0), + key_cache.stride(1), + key_cache.stride(2), + value_cache.stride(0), + value_cache.stride(1), + value_cache.stride(2), + conv_block_table.stride(0), + conv_block_table.shape[1], + conv_block_size, + key_cache.shape[1], + Q_WIDTH=num_q_heads * head_dim, + KV_WIDTH=num_kv_heads * head_dim, + NUM_KV_HEADS=num_kv_heads, + HEAD_DIM=head_dim, + WINDOW_SIZE=k_weight.shape[1], + OFF_K=off_k, + OFF_V=off_v, + BLOCK_ROWS=KV_BLOCK_ROWS, + num_warps=KV_NUM_WARPS, + ) + + +def _run_fused_small( + qkvr: torch.Tensor, + q_norm_weight: torch.Tensor, + q_out: torch.Tensor, + rel_proj: torch.Tensor, + rel_out: torch.Tensor, + k_weight: torch.Tensor, + v_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + conv_cache: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + positions: torch.Tensor, + seq_idx: torch.Tensor, + conv_slot_mapping: torch.Tensor, + conv_block_table: torch.Tensor, + query_start: torch.Tensor, + attention_slot_mapping: torch.Tensor, + *, + eps: float, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + off_k: int, + off_v: int, + conv_block_size: int, + log_scaling: torch.Tensor | None, +) -> None: + tokens = qkvr.shape[0] + + num_q_rows = tokens * num_q_heads + grid = (num_q_rows + tokens * num_kv_heads,) + _qkvr_qkv_kernel[grid]( + qkvr, + q_norm_weight, + q_out, + rel_proj, + rel_out, + k_weight, + v_weight, + k_norm_weight, + conv_cache, + key_cache, + value_cache, + positions, + seq_idx, + conv_slot_mapping, + conv_block_table, + query_start, + attention_slot_mapping, + log_scaling if log_scaling is not None else positions, + tokens, + eps, + qkvr.stride(0), + conv_cache.stride(0), + conv_cache.stride(1), + conv_cache.stride(2), + conv_cache.stride(3), + key_cache.stride(0), + key_cache.stride(1), + key_cache.stride(2), + value_cache.stride(0), + value_cache.stride(1), + value_cache.stride(2), + conv_block_table.stride(0), + conv_block_table.shape[1], + conv_block_size, + key_cache.shape[1], + Q_WIDTH=num_q_heads * head_dim, + KV_WIDTH=num_kv_heads * head_dim, + NUM_Q_HEADS=num_q_heads, + NUM_KV_HEADS=num_kv_heads, + HEAD_DIM=head_dim, + WINDOW_SIZE=k_weight.shape[1], + OFF_K=off_k, + OFF_V=off_v, + APPLY_LOG_SCALING=log_scaling is not None, + D_REL=16, + REL_EXTENT=rel_proj.shape[1], + REL_EXTENT_PADDED=triton.next_power_of_2(rel_proj.shape[1]), + num_warps=SMALL_NUM_WARPS, + ) + + +def fused_qkvr_prep( + qkvr: torch.Tensor, + k_weight: torch.Tensor, + v_weight: torch.Tensor, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + rel_proj: torch.Tensor, + eps: float, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + d_rel: int, + conv_cache: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + positions: torch.Tensor, + conv_block_table: torch.Tensor, + seq_idx: torch.Tensor, + conv_slot_mapping: torch.Tensor, + query_start: torch.Tensor, + attention_slot_mapping: torch.Tensor, + off_k: int, + off_v: int, + conv_block_size: int, + log_scaling: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + assert d_rel == 16 and rel_proj.shape[0] == 16 + assert head_dim == 128 + assert qkvr.is_contiguous() + assert k_weight.stride() == (k_weight.shape[1], 1) + assert v_weight.stride() == (v_weight.shape[1], 1) + assert rel_proj.stride() == (rel_proj.shape[1], 1) + assert conv_cache.stride(3) == 1 + assert key_cache.stride(3) == 1 and value_cache.stride(3) == 1 + tokens = qkvr.shape[0] + q_out = torch.empty( + (tokens, num_q_heads * head_dim), dtype=qkvr.dtype, device=qkvr.device + ) + rel_out = torch.empty( + (tokens, num_q_heads, rel_proj.shape[1]), + dtype=qkvr.dtype, + device=qkvr.device, + ) + if tokens == 0: + return q_out, rel_out + + if tokens < SMALL_TOKEN_THRESHOLD: + _run_fused_small( + qkvr, + q_norm_weight, + q_out, + rel_proj, + rel_out, + k_weight, + v_weight, + k_norm_weight, + conv_cache, + key_cache, + value_cache, + positions, + seq_idx, + conv_slot_mapping, + conv_block_table, + query_start, + attention_slot_mapping, + eps=eps, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + off_k=off_k, + off_v=off_v, + conv_block_size=conv_block_size, + log_scaling=log_scaling, + ) + return q_out, rel_out + + kv_stream = aux_stream() + assert kv_stream is not None + current_stream = torch.cuda.current_stream() + kv_stream.wait_stream(current_stream) + with torch.cuda.stream(kv_stream): + _run_tiled_kv( + qkvr, + k_weight, + v_weight, + k_norm_weight, + conv_cache, + key_cache, + value_cache, + positions, + seq_idx, + conv_slot_mapping, + conv_block_table, + query_start, + attention_slot_mapping, + eps=eps, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + off_k=off_k, + off_v=off_v, + conv_block_size=conv_block_size, + ) + _run_tiled_q( + qkvr, + q_norm_weight, + q_out, + positions, + eps=eps, + num_q_heads=num_q_heads, + head_dim=head_dim, + log_scaling=log_scaling, + ) + qkvr_rel_proj( + qkvr, + rel_proj, + rel_out, + log_scaling, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + d_rel=d_rel, + ) + current_stream.wait_stream(kv_stream) + return q_out, rel_out diff --git a/vllm/models/inkling/amd/ops/rel_attention_decode.py b/vllm/models/inkling/amd/ops/rel_attention_decode.py new file mode 100644 index 0000000000..810eb9cb5e --- /dev/null +++ b/vllm/models/inkling/amd/ops/rel_attention_decode.py @@ -0,0 +1,403 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 LightSeek Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +"""Split-KV decode for Inkling relative attention on ROCm. + +Adapted from LightSeek TokenSpeed's portable Triton relative-MHA decode. +""" + +from __future__ import annotations + +import os + +import torch + +from vllm.triton_utils import tl, triton + +_MIN_BLOCK_KV = tl.constexpr(32) + + +def decode_split_count(max_kv_len: int, window_left: int) -> int: + """Return the number of parallel KV partitions for decode.""" + if window_left >= 0: + effective_len = max(1, min(max_kv_len, window_left + 1)) + return min(4, max(1, triton.cdiv(effective_len, 128))) + return min(32, max(1, triton.cdiv(max(1, max_kv_len), 2048))) + + +def use_split_kv_decode( + *, + max_query_len: int, + max_kv_len: int, + page_size: int, + window_left: int, +) -> bool: + """Select split-KV only where it outperforms the single-pass kernel.""" + if os.getenv("INKLING_SPLIT_KV", "1") != "1": + return False + if max_query_len != 1: + return False + if page_size >= 64: + return True + if window_left >= 0: + return page_size >= 32 + return max_kv_len >= 8192 + + +@triton.jit +def _split_kv_stage1( + q_ptr, + rel_ptr, + k_ptr, + v_ptr, + block_table_ptr, + cache_seqlens, + mid_out_ptr, + mid_lse_ptr, + softmax_scale, + stride_q_t, + stride_q_h, + stride_k_b, + stride_k_p, + stride_k_h, + stride_k_d, + stride_v_b, + stride_v_p, + stride_v_h, + stride_v_d, + stride_r_t, + stride_r_h, + stride_r_e, + stride_mo_t, + stride_mo_h, + stride_mo_s, + stride_ml_t, + stride_ml_h, + stride_ml_s, + stride_bt_b: tl.constexpr, + page_size: tl.constexpr, + window_left: tl.constexpr, + gqa_group_size: tl.constexpr, + num_q_heads: tl.constexpr, + head_dim: tl.constexpr, + rel_extent: tl.constexpr, + max_kv_splits: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_N: tl.constexpr, +): + pid_t = tl.program_id(0) + pid_h = tl.program_id(1) + pid_s = tl.program_id(2) + + head_blocks_per_kv = tl.cdiv(gqa_group_size, BLOCK_H) + kv_head = pid_h // head_blocks_per_kv + valid_block_h: tl.constexpr = min(BLOCK_H, gqa_group_size) + off_h = pid_h * valid_block_h + tl.arange(0, BLOCK_H) + h_valid = (off_h < (pid_h + 1) * valid_block_h) & (off_h < num_q_heads) + off_d = tl.arange(0, BLOCK_D) + d_valid = off_d < head_dim + + cache_len = tl.load(cache_seqlens + pid_t) + effective_len = ( + tl.minimum(cache_len, window_left + 1) if window_left >= 0 else cache_len + ) + kv_offset = cache_len - effective_len + split_len = ( + tl.cdiv(tl.cdiv(effective_len, max_kv_splits), _MIN_BLOCK_KV) * _MIN_BLOCK_KV + ) + split_start = split_len * pid_s + split_end = tl.minimum(split_start + split_len, effective_len) + + q_offsets = pid_t * stride_q_t + off_h[:, None] * stride_q_h + off_d[None, :] + row_max = tl.full((BLOCK_H,), float("-inf"), dtype=tl.float32) + row_sum = tl.zeros((BLOCK_H,), dtype=tl.float32) + acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32) + + if split_end > split_start: + q = tl.load( + q_ptr + q_offsets, + mask=h_valid[:, None] & d_valid[None, :], + other=0.0, + ) + for start_n in range(split_start, split_end, BLOCK_N): + off_n = start_n + tl.arange(0, BLOCK_N) + n_valid = off_n < split_end + token_idx = kv_offset + off_n + logical_page = token_idx // page_size + page_offset = token_idx % page_size + physical_page = tl.load( + block_table_ptr + pid_t * stride_bt_b + logical_page, + mask=n_valid, + other=0, + ) + k_offsets = ( + physical_page[None, :].to(tl.int64) * stride_k_b + + page_offset[None, :] * stride_k_p + + kv_head * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_ptr + k_offsets, + mask=d_valid[:, None] & n_valid[None, :], + other=0.0, + ) + scores = tl.dot(q, k.to(q.dtype)) * softmax_scale + + rel_dist = cache_len - 1 - token_idx + rel_valid = (rel_dist >= 0) & (rel_dist < rel_extent) + rel_idx = tl.maximum(0, tl.minimum(rel_dist, rel_extent - 1)) + rel_offsets = ( + pid_t * stride_r_t + + off_h[:, None] * stride_r_h + + rel_idx[None, :] * stride_r_e + ) + rel_bias = tl.load( + rel_ptr + rel_offsets, + mask=h_valid[:, None] & rel_valid[None, :] & n_valid[None, :], + other=0.0, + ) + scores += rel_bias.to(tl.float32) + scores = tl.where( + h_valid[:, None] & n_valid[None, :], + scores, + float("-inf"), + ) + + v_offsets = ( + physical_page[:, None].to(tl.int64) * stride_v_b + + page_offset[:, None] * stride_v_p + + kv_head * stride_v_h + + off_d[None, :] * stride_v_d + ) + v = tl.load( + v_ptr + v_offsets, + mask=n_valid[:, None] & d_valid[None, :], + other=0.0, + ) + + next_max = tl.maximum(tl.max(scores, axis=1), row_max) + old_scale = tl.exp(row_max - next_max) + probs = tl.exp(scores - next_max[:, None]) + acc *= old_scale[:, None] + acc += tl.dot(probs.to(v.dtype), v) + row_sum = row_sum * old_scale + tl.sum(probs, axis=1) + row_max = next_max + + mid_out_offsets = ( + pid_t * stride_mo_t + + off_h[:, None] * stride_mo_h + + pid_s * stride_mo_s + + off_d[None, :] + ) + tl.store( + mid_out_ptr + mid_out_offsets, + acc / row_sum[:, None], + mask=h_valid[:, None] & d_valid[None, :], + ) + mid_lse_offsets = ( + pid_t * stride_ml_t + off_h * stride_ml_h + pid_s * stride_ml_s + ) + tl.store( + mid_lse_ptr + mid_lse_offsets, + row_max + tl.log(row_sum), + mask=h_valid, + ) + + +@triton.jit +def _split_kv_stage2( + mid_out_ptr, + mid_lse_ptr, + out_ptr, + cache_seqlens, + stride_mo_t, + stride_mo_h, + stride_mo_s, + stride_ml_t, + stride_ml_h, + stride_ml_s, + stride_o_t, + stride_o_h, + window_left: tl.constexpr, + head_dim: tl.constexpr, + max_kv_splits: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid_t = tl.program_id(0) + pid_h = tl.program_id(1) + + cache_len = tl.load(cache_seqlens + pid_t) + effective_len = ( + tl.minimum(cache_len, window_left + 1) if window_left >= 0 else cache_len + ) + split_len = ( + tl.cdiv(tl.cdiv(effective_len, max_kv_splits), _MIN_BLOCK_KV) * _MIN_BLOCK_KV + ) + + off_d = tl.arange(0, BLOCK_D) + d_valid = off_d < head_dim + row_max = -float("inf") + row_sum = 0.0 + acc = tl.zeros((BLOCK_D,), dtype=tl.float32) + + for split_id in range(max_kv_splits): + split_start = split_len * split_id + split_end = tl.minimum(split_start + split_len, effective_len) + if split_end > split_start: + value = tl.load( + mid_out_ptr + + pid_t * stride_mo_t + + pid_h * stride_mo_h + + split_id * stride_mo_s + + off_d, + mask=d_valid, + other=0.0, + ) + split_lse = tl.load( + mid_lse_ptr + + pid_t * stride_ml_t + + pid_h * stride_ml_h + + split_id * stride_ml_s + ) + next_max = tl.maximum(split_lse, row_max) + old_scale = tl.exp(row_max - next_max) + split_scale = tl.exp(split_lse - next_max) + acc = acc * old_scale + value * split_scale + row_sum = row_sum * old_scale + split_scale + row_max = next_max + + tl.store( + out_ptr + pid_t * stride_o_t + pid_h * stride_o_h + off_d, + acc / row_sum, + mask=d_valid, + ) + + +@torch.no_grad() +def inkling_rel_attention_split_kv_decode( + q: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + *, + block_table: torch.Tensor, + cache_seqlens: torch.Tensor, + softmax_scale: float, + window_left: int, + rel_extent: int, + rel_logits: torch.Tensor, + max_kv_len: int, + out: torch.Tensor, +) -> torch.Tensor: + """Run split-KV relative attention for single-token decode.""" + num_kv_heads = key_cache.shape[2] + gqa_group_size = q.shape[1] // num_kv_heads + max_kv_splits = decode_split_count(max_kv_len, window_left) + block_d = triton.next_power_of_2(q.shape[2]) + block_h = min(8, gqa_group_size) + + mid_out = torch.empty( + q.shape[0], + q.shape[1], + max_kv_splits, + q.shape[2], + dtype=torch.float32, + device=q.device, + ) + mid_lse = torch.empty( + q.shape[0], + q.shape[1], + max_kv_splits, + dtype=torch.float32, + device=q.device, + ) + stage1_grid = ( + q.shape[0], + triton.cdiv(q.shape[1], block_h), + max_kv_splits, + ) + _split_kv_stage1[stage1_grid]( + q, + rel_logits, + key_cache, + value_cache, + block_table, + cache_seqlens, + mid_out, + mid_lse, + softmax_scale, + q.stride(0), + q.stride(1), + key_cache.stride(0), + key_cache.stride(1), + key_cache.stride(2), + key_cache.stride(3), + value_cache.stride(0), + value_cache.stride(1), + value_cache.stride(2), + value_cache.stride(3), + rel_logits.stride(0), + rel_logits.stride(1), + rel_logits.stride(2), + mid_out.stride(0), + mid_out.stride(1), + mid_out.stride(2), + mid_lse.stride(0), + mid_lse.stride(1), + mid_lse.stride(2), + block_table.stride(0), + page_size=key_cache.shape[1], + window_left=window_left, + gqa_group_size=gqa_group_size, + num_q_heads=q.shape[1], + head_dim=q.shape[2], + rel_extent=rel_extent, + max_kv_splits=max_kv_splits, + BLOCK_D=block_d, + BLOCK_H=block_h, + BLOCK_N=key_cache.shape[1], + num_warps=4, + num_stages=1, + ) + stage2_grid = (q.shape[0], q.shape[1]) + _split_kv_stage2[stage2_grid]( + mid_out, + mid_lse, + out, + cache_seqlens, + mid_out.stride(0), + mid_out.stride(1), + mid_out.stride(2), + mid_lse.stride(0), + mid_lse.stride(1), + mid_lse.stride(2), + out.stride(0), + out.stride(1), + window_left=window_left, + head_dim=q.shape[2], + max_kv_splits=max_kv_splits, + BLOCK_D=block_d, + num_warps=4, + num_stages=2, + ) + return out diff --git a/vllm/models/inkling/amd/ops/sconv.py b/vllm/models/inkling/amd/ops/sconv.py new file mode 100644 index 0000000000..7ebb2a8a0a --- /dev/null +++ b/vllm/models/inkling/amd/ops/sconv.py @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling short-convolution kernels backed by a paged sliding-window state cache. + +Each layer's 4 conv streams (K, V, attn-output, mlp-output) share one paged KV +cache ``[num_blocks, H, N, D]`` (head-major; see ``sconv_swa_attn.py``). A stream +occupies the contiguous D-sub-range ``[off_s, off_s + ws)`` across all ``H`` +heads, so its flat per-token width is ``H * ws`` and that is the conv channel +dim. The cache stores the conv *input* at every absolute position. + +``fused_sconv`` is the single-launch path used by the model: per token it writes +the current input to its slot and convolves the ``W`` taps ending at its +absolute position. A tap landing inside the current forward is read from the +immutable input ``x`` (row ``src - pos + pid_t``); only pre-forward taps are read +from the paged cache (window position ``src`` -> physical block via +``block_table[req, src // N]``). The just-written slot is never read back this +step, so there is no write/read hazard within or across programs -- which is +why this needs no decode-vs-prefill split and is valid for prefill / decode / +spec alike. + +All kernels address the cache purely by ``(slot, absolute_position)`` and +allocate nothing inside the captured region; their grids depend only on the +token count (``fused_sconv`` on a fixed token/channel tiling), so the same +path replays correctly under eager, breakable PIECEWISE, and FULL cudagraphs +without any data-dependent shape or branch. +""" + +from __future__ import annotations + +import torch + +from vllm.triton_utils import tl, triton + + +@triton.jit +def _fused_sconv_kernel( + x_ptr, # [T, H*WS] head-major current-token inputs (also residual) + cache_ptr, # [num_blocks, H, N, D] paged (page-strided view) + weight_ptr, # [H*WS, W] + out_ptr, # [T, H*WS] + pos_ptr, # [T] int64 absolute position per token + seq_idx_ptr, # [T] int32 token -> batch request + slot_ptr, # [T] int64 flat slot (block*N + blk_off); < 0 => PAD + block_table_ptr, # [num_reqs, max_blocks] int32 block_table + qstart_ptr, # [T] int32 first x-row of the token's request + T, # num tokens + stride_x_t, + stride_c_blk, + stride_c_h, + stride_c_n, + stride_c_d, + stride_w_d, + stride_w_w, + stride_bt_r, + MAX_BLOCKS, + N, # block_size + W: tl.constexpr, + USE_SILU: tl.constexpr, + USE_RESIDUAL: tl.constexpr, + OFF_S: tl.constexpr, + WS: tl.constexpr, + H: tl.constexpr, + BT: tl.constexpr, + BLOCK_C: tl.constexpr, +): + # Each program owns a [BT tokens, BLOCK_C channels] tile. The flat channel + # index packs all H heads head-major (head = c // WS, in-stream offset = + # c % WS), so one program spans heads -- no per-head launch and no + # next_power_of_2(WS) lane waste. + pid_t = tl.program_id(0) + pid_c = tl.program_id(1) + toff = pid_t * BT + tl.arange(0, BT) # [BT] token rows + coff = pid_c * BLOCK_C + tl.arange(0, BLOCK_C) # [BLOCK_C] flat channels + C = H * WS + t_mask = toff < T + c_mask = coff < C + head = tl.minimum(coff // WS, H - 1) # clamp keeps masked lanes in-buffer + cd = OFF_S + coff % WS # cache D-index of the channel's stream slot + + slot = tl.load(slot_ptr + toff, mask=t_mask, other=-1) # [BT] + valid = slot >= 0 + pos = tl.load(pos_ptr + toff, mask=t_mask, other=0) + req = tl.load(seq_idx_ptr + toff, mask=t_mask, other=0) + qstart = tl.load(qstart_ptr + toff, mask=t_mask, other=0) + + tc_mask = t_mask[:, None] & c_mask[None, :] + + # 1) Insert each token's input into its paged slot (skip PAD rows). + xv = tl.load(x_ptr + toff[:, None] * stride_x_t + coff[None, :], mask=tc_mask) + safe_slot = tl.maximum(slot, 0) + dst = ( + cache_ptr + + (safe_slot // N)[:, None] * stride_c_blk + + head[None, :] * stride_c_h + + (safe_slot % N)[:, None] * stride_c_n + + cd[None, :] * stride_c_d + ) + tl.store(dst, xv, mask=tc_mask & valid[:, None]) + + # 2) Convolve the W taps ending at each token's `pos`. Each tap is read from + # `x` if it falls inside this forward (row >= the request's first row), else + # from the paged cache. Exactly one source is unmasked per tap, so we sum both. + acc = tl.zeros([BT, BLOCK_C], dtype=tl.float32) + for iw in tl.static_range(W): + src = pos - (W - 1) + iw # [BT] absolute window position + row = toff - (W - 1) + iw # [BT] x-row of `src` (== src - pos + token) + in_win = valid & (src >= 0) + intra = in_win & (row >= qstart) + cached = in_win & (row < qstart) + # intra-forward tap: read the immutable input x (never the slot we just + # wrote), so there is no write/read hazard. + safe_row = tl.maximum(row, 0) + xt = tl.load( + x_ptr + safe_row[:, None] * stride_x_t + coff[None, :], + mask=c_mask[None, :] & intra[:, None], + other=0.0, + ).to(tl.float32) + # pre-forward tap: read from the paged cache via the block table. Clamp + # addressing terms; the load is masked off when out of window. + safe_src = tl.maximum(src, 0) + safe_lblk = tl.minimum(safe_src // N, MAX_BLOCKS - 1) + blk = tl.load( + block_table_ptr + req * stride_bt_r + safe_lblk, mask=cached, other=0 + ).to(tl.int64) + cbase = ( + cache_ptr + + blk[:, None] * stride_c_blk + + head[None, :] * stride_c_h + + (safe_src % N)[:, None] * stride_c_n + + cd[None, :] * stride_c_d + ) + cv = tl.load(cbase, mask=c_mask[None, :] & cached[:, None], other=0.0).to( + tl.float32 + ) + wv = tl.load( + weight_ptr + coff * stride_w_d + iw * stride_w_w, mask=c_mask, other=0.0 + ).to(tl.float32) + acc += (xt + cv) * wv[None, :] + + if USE_SILU: + acc = acc * tl.sigmoid(acc) + if USE_RESIDUAL: + acc += xv.to(tl.float32) + + tl.store( + out_ptr + toff[:, None] * stride_x_t + coff[None, :], + acc.to(out_ptr.dtype.element_ty), + mask=tc_mask, + ) + + +def fused_sconv( + x: torch.Tensor, # [T, H*ws] head-major current-token inputs + weight: torch.Tensor, # [H*ws, W] + cache: torch.Tensor, # [num_blocks, H, N, D] paged + positions: torch.Tensor, # [T] int64 absolute position per token + block_table: torch.Tensor, # [num_reqs, max_blocks] int32 + seq_idx: torch.Tensor, # [T] int32 token -> batch request + slot_mapping: torch.Tensor, # [T] int64 flat slot (PAD = -1 => skip) + query_start: torch.Tensor, # [T] int32 first x-row of the token's request + off_s: int, + ws: int, + block_size: int, + activation: str | None = None, + use_residual: bool = True, +) -> torch.Tensor: + """Single-launch insert + depthwise causal conv1d over the paged cache. + + Reads same-forward taps from ``x`` and pre-forward taps from the cache, so + it is race-free in one launch for prefill / decode / spec and cudagraph-safe + under eager / piecewise / full capture. + """ + T = x.shape[0] + out = torch.empty_like(x) + if T == 0: + return out + assert x.is_contiguous() + assert cache.stride(3) == 1, "cache D-dim must be contiguous" + H = cache.shape[1] + W = weight.shape[1] + C = H * ws # flat conv channel dim (all heads, head-major) + # Tile BT tokens x BLOCK_C channels per program: enough work per CTA to + # amortize the per-token addressing, while keeping the grid large for + # prefill. BLOCK_C spans heads so there is no per-head launch. A ~2K-element + # tile at 4 warps measured best on Blackwell; larger tiles spill registers. + BLOCK_C = min(triton.next_power_of_2(C), 256) + BT = 8 + grid = (triton.cdiv(T, BT), triton.cdiv(C, BLOCK_C)) + _fused_sconv_kernel[grid]( + x, + cache, + weight, + out, + positions, + seq_idx, + slot_mapping, + block_table, + query_start, + T, + x.stride(0), + cache.stride(0), + cache.stride(1), + cache.stride(2), + cache.stride(3), + weight.stride(0), + weight.stride(1), + block_table.stride(0), + block_table.shape[1], + block_size, + W=W, + USE_SILU=activation in ("silu", "swish"), + USE_RESIDUAL=use_residual, + OFF_S=off_s, + WS=ws, + H=H, + BT=BT, + BLOCK_C=BLOCK_C, + num_warps=4, + ) + return out + + +@triton.jit +def _seq_metadata_kernel( + qsl_ptr, # [num_reqs + 1] int32 cumulative query start rows + seq_idx_ptr, # [T] int32 out: token -> owning request + query_start_ptr, # [T] int32 out: first x-row of the token's request + num_reqs, + num_actual_tokens, + num_padded_tokens, + n_iters, # ceil(log2(num_reqs)): binary-search depth + BLOCK: tl.constexpr, +): + offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + tok = offs.to(tl.int32) + # Largest j in [0, num_reqs) with qsl[j] <= tok. + lo = tl.zeros([BLOCK], tl.int32) + hi = tl.full([BLOCK], num_reqs - 1, tl.int32) + for _ in range(n_iters): + mid = (lo + hi + 1) // 2 + below = tl.load(qsl_ptr + mid) <= tok + lo = tl.where(below, mid, lo) + hi = tl.where(below, hi, mid - 1) + actual = offs < num_actual_tokens + padded = offs < num_padded_tokens + query_start = tl.load(qsl_ptr + lo) + tl.store(seq_idx_ptr + offs, tl.where(actual, lo, 0), mask=padded) + tl.store( + query_start_ptr + offs, + tl.where(actual, query_start, 0), + mask=padded, + ) + + +def sconv_seq_metadata( + query_start_loc: torch.Tensor, + num_reqs: int, + num_actual_tokens: int, + seq_idx_out: torch.Tensor, + query_start_out: torch.Tensor, + num_padded_tokens: int | None = None, +) -> None: + """Fill static per-token seq_idx / query_start buffers in one launch. + + Replaces the arange + searchsorted + clamp + gather + 2x copy chain of the + sconv metadata build with a single kernel writing both persistent buffers. + Padded rows are filled with zero and must have ``slot_mapping == -1``. + """ + if num_padded_tokens is None: + num_padded_tokens = num_actual_tokens + if num_padded_tokens < num_actual_tokens: + raise ValueError("num_padded_tokens must cover all actual tokens") + if num_padded_tokens > seq_idx_out.shape[0]: + raise ValueError("seq_idx_out is too small for the padded token count") + if num_padded_tokens > query_start_out.shape[0]: + raise ValueError("query_start_out is too small for the padded token count") + + BLOCK = 256 + n_iters = (num_reqs - 1).bit_length() + grid = (triton.cdiv(num_padded_tokens, BLOCK),) + _seq_metadata_kernel[grid]( + query_start_loc, + seq_idx_out, + query_start_out, + num_reqs, + num_actual_tokens, + num_padded_tokens, + n_iters, + BLOCK=BLOCK, + ) diff --git a/vllm/models/inkling/amd/ops/silu_and_mul.py b/vllm/models/inkling/amd/ops/silu_and_mul.py new file mode 100644 index 0000000000..42ec25f632 --- /dev/null +++ b/vllm/models/inkling/amd/ops/silu_and_mul.py @@ -0,0 +1,197 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SwiGLU kernels for the Inkling MLP layers. + +``silu_and_mul_triton``: SiLU-and-mul over the checkpoint's interleaved +fused gate/up layout (dense MLP). ``sink_silu_mul_epilogue``: the sink-expert +variant with the per-expert dequant scale and per-token gamma fused in. +""" + +import torch + +from vllm.triton_utils import tl, triton + + +@triton.jit(do_not_specialize=["M"]) +def _silu_and_mul_triton_kernel( + gateup_out_ptr, + down_inp_ptr, + M, + N: tl.constexpr, + GRID_SIZE: tl.constexpr, + NUM_STAGES: tl.constexpr, + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + EVEN_N: tl.constexpr, + INT64_INDEX: tl.constexpr, +): + start_pid = tl.program_id(0) + if INT64_INDEX: + start_pid = start_pid.to(tl.int64) + M = M.to(tl.int64) + + NUM_BLOCKS_N: tl.constexpr = tl.cdiv(N, BLOCK_SIZE_N) + num_blocks_mn = tl.cdiv(M, BLOCK_SIZE_M) * NUM_BLOCKS_N + + for pid in tl.range(start_pid, num_blocks_mn, GRID_SIZE, num_stages=NUM_STAGES): + pid_m = pid // NUM_BLOCKS_N + pid_n = pid % NUM_BLOCKS_N + + offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_n = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + mask_m = offs_m < M + mask_n = offs_n < N + + # Interleaved fused gate/up: [g0, u0, g1, u1, ...]. + mask_offs_2n = pid_n * BLOCK_SIZE_N + tl.arange(0, 2 * BLOCK_SIZE_N) // 2 + tl.static_assert(BLOCK_SIZE_N % 8 == 0, f"{BLOCK_SIZE_N=}") + mask_2n = mask_offs_2n < N + mask_2n = tl.max_constancy(mask_2n, [16]) + + offs_2n = pid_n * 2 * BLOCK_SIZE_N + tl.arange(0, 2 * BLOCK_SIZE_N) + offs_m2n = offs_m[:, None] * N * 2 + offs_2n[None, :] + + if EVEN_N or pid_n * BLOCK_SIZE_N + BLOCK_SIZE_N <= N: + gateup_out = tl.load( + gateup_out_ptr + offs_m2n, mask=mask_m[:, None], other=0.0 + ) + else: + mask_m2n = mask_m[:, None] & mask_2n[None, :] + gateup_out = tl.load(gateup_out_ptr + offs_m2n, mask=mask_m2n, other=0.0) + + gate_out, up_out = tl.split( + tl.reshape(gateup_out, (BLOCK_SIZE_M, BLOCK_SIZE_N, 2)) + ) + gate_out = gate_out.to(tl.float32) + up_out = up_out.to(tl.float32) + + down_inp = gate_out * tl.sigmoid(gate_out) * up_out + + mask_mn = mask_m[:, None] if EVEN_N else mask_m[:, None] & mask_n[None, :] + offs_mn = offs_m[:, None] * N + offs_n[None, :] + tl.store(down_inp_ptr + offs_mn, down_inp, mask=mask_mn) + + +def silu_and_mul_triton(gateup_output: torch.Tensor) -> torch.Tensor: + """SiLU-and-mul for the interleaved fused gate/up layout. + + Adapted from ``inkling_kernels.activation.silu_and_mul_fwd`` (without MXFP). + """ + assert gateup_output.is_contiguous(), ( + f"{gateup_output.shape=} {gateup_output.stride()=}" + ) + assert gateup_output.ndim == 2, f"{gateup_output.shape=}" + + M = gateup_output.shape[0] + hidden_size = gateup_output.shape[1] + assert hidden_size % 2 == 0, f"{hidden_size=}" + N = hidden_size // 2 + + down_input = torch.empty( + (M, N), device=gateup_output.device, dtype=gateup_output.dtype + ) + if M == 0: + return down_input + + BLOCK_SIZE_N = max(8, min(256, triton.next_power_of_2(N))) + if M <= 1: + BLOCK_SIZE_M = 4 + elif M <= 256: + BLOCK_SIZE_M = 2 + elif M < 4096: + BLOCK_SIZE_M = 4 + else: + BLOCK_SIZE_M = 16 + BLOCK_SIZE_N = max(8, min(128, triton.next_power_of_2(N))) + max_grid_size = triton.cdiv(M, BLOCK_SIZE_M) * triton.cdiv(N, BLOCK_SIZE_N) + num_sms = torch.cuda.get_device_properties( + gateup_output.device + ).multi_processor_count + grid_size = min(num_sms * 4, max_grid_size) + + _silu_and_mul_triton_kernel[(grid_size,)]( + gateup_out_ptr=gateup_output, + down_inp_ptr=down_input, + M=M, + N=N, + GRID_SIZE=grid_size, + NUM_STAGES=1, + BLOCK_SIZE_M=BLOCK_SIZE_M, + BLOCK_SIZE_N=BLOCK_SIZE_N, + EVEN_N=N % BLOCK_SIZE_N == 0, + INT64_INDEX=gateup_output.nbytes >= 2**31, + num_warps=8, + ) + + return down_input + + +@triton.jit(do_not_specialize=["T"]) +def _sink_epilogue_kernel( + raw_ptr, # [T, S * 2F] gemm1 output, interleaved g/u pairs per expert block + alpha_ptr, # [S] fp32 per-expert pre-SiLU dequant scale + gamma_ptr, # [T, S] fp32 per-token sink weights (may be strided) + ratio_ptr, # [S] fp32 per-expert post-SiLU scale (gemm2 alpha ratio) + out_ptr, # [T, S * F] output + T, + stride_raw_0, + stride_gamma_0, + F: tl.constexpr, + S: tl.constexpr, + BLOCK_F: tl.constexpr, +): + pid_t = tl.program_id(0).to(tl.int64) + pid_sf = tl.program_id(1) + if pid_t >= T: + return + s = pid_sf // (F // BLOCK_F) + offs_f = (pid_sf % (F // BLOCK_F)) * BLOCK_F + tl.arange(0, BLOCK_F) + + base = pid_t * stride_raw_0 + s * 2 * F + gate = tl.load(raw_ptr + base + 2 * offs_f).to(tl.float32) + up = tl.load(raw_ptr + base + 2 * offs_f + 1).to(tl.float32) + alpha = tl.load(alpha_ptr + s) + weight = tl.load(gamma_ptr + pid_t * stride_gamma_0 + s) * tl.load(ratio_ptr + s) + + gate *= alpha + up *= alpha + h = gate * tl.sigmoid(gate) * up * weight + tl.store(out_ptr + pid_t * (S * F) + s * F + offs_f, h) + + +def sink_silu_mul_epilogue( + raw: torch.Tensor, # [T, S * 2F] gemm1 output (interleaved gate/up rows) + alphas: torch.Tensor, # [S] fp32 + gammas: torch.Tensor, # [T, S] fp32 + ratios: torch.Tensor, # [S] fp32 + n_experts: int, + out_dtype: torch.dtype, +) -> torch.Tensor: + """Fused sink-expert epilogue: silu(g * a_e) * (u * a_e) * (gamma * r_e). + + One kernel replaces the per-expert dequant column scale, the SwiGLU, and + the per-token gamma multiply between the two sink GEMMs. + """ + tokens = raw.shape[0] + f = raw.shape[1] // (2 * n_experts) + out = torch.empty((tokens, n_experts * f), device=raw.device, dtype=out_dtype) + if tokens == 0: + return out + # raw may be a column-slice of a padded GEMM output (rows strided). + assert raw.stride(1) == 1 and gammas.stride(1) == 1 + # Largest power-of-two divisor of f (f = 768 -> 256), capped at 512. + block_f = min(512, f & (-f)) + _sink_epilogue_kernel[(tokens, n_experts * (f // block_f))]( + raw, + alphas, + gammas, + ratios, + out, + tokens, + raw.stride(0), + gammas.stride(0), + F=f, + S=n_experts, + BLOCK_F=block_f, + ) + return out diff --git a/vllm/models/inkling/amd/sconv_swa_attn.py b/vllm/models/inkling/amd/sconv_swa_attn.py new file mode 100644 index 0000000000..7b8958d556 --- /dev/null +++ b/vllm/models/inkling/amd/sconv_swa_attn.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling short-conv state managed as a sliding-window KV cache. + +Each decoder layer owns one ``InklingConvState`` (an ``AttentionLayerBase``) that +emits a single ``SlidingWindowSpec`` for the layer's 4 sconv streams (K, V, +attn-output, mlp-output), packed head-major into one block: + + H = num_kv_heads (per-rank), N = block_size = sconv_kernel_size, + D = head_dim(K) + head_dim(V) + hidden/H(attn) + hidden/H(mlp) + +``D`` is TP-invariant; per rank we store ``H/TP`` heads of width ``D``. The conv +reads/writes this cache out-of-band via a custom backend; the (smaller) conv page +is padded up to the uniform attention page by ``unify_kv_cache_spec_page_size``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ClassVar + +import torch +from torch import nn + +from vllm.config import VllmConfig, get_current_vllm_config +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.v1.attention.backend import ( + AttentionBackend, + AttentionCGSupport, + AttentionMetadata, + AttentionMetadataBuilder, + CommonAttentionMetadata, +) +from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, SlidingWindowSpec + +from .ops.sconv import sconv_seq_metadata + +# Stream order within the per-head packed D (== contiguous sub-ranges). +_K, _V, _ATTN, _MLP = 0, 1, 2, 3 + + +@dataclass +class InklingSconvMetadata(AttentionMetadata): + block_table: torch.Tensor # [num_reqs, max_blocks] physical blocks per req + slot_mapping: torch.Tensor # [T] int64 flat slot of each token (-1 => skip) + seq_idx: torch.Tensor # [T] int32 token -> batch request + query_start: torch.Tensor # [T] int32 first x-row of each token's request + + +class InklingSconvMetadataBuilder(AttentionMetadataBuilder[InklingSconvMetadata]): + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + + def __init__( + self, + kv_cache_spec: AttentionSpec, + layer_names: list[str], + vllm_config: VllmConfig, + device: torch.device, + ) -> None: + super().__init__(kv_cache_spec, layer_names, vllm_config, device) + assert isinstance(kv_cache_spec, SlidingWindowSpec) + # Persistent per-token buffers for CUDA graph capture. + max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens + self.seq_idx_buffer = torch.empty( + max_num_tokens, dtype=torch.int32, device=device + ) + self.query_start_buffer = torch.empty( + max_num_tokens, dtype=torch.int32, device=device + ) + + def build( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + ) -> InklingSconvMetadata: + num_reqs = common_attn_metadata.num_reqs + num_actual_tokens = int(common_attn_metadata.query_start_loc_cpu[-1]) + num_padded_tokens = common_attn_metadata.slot_mapping.shape[0] + assert num_padded_tokens >= num_actual_tokens + + # Per-token seq_idx (owning request) and query_start (first x-row of + # that request; the fused kernel uses it to tell same-forward taps, + # read from x, from pre-forward taps, read from cache) in one launch. + sconv_seq_metadata( + common_attn_metadata.query_start_loc, + num_reqs, + num_actual_tokens, + self.seq_idx_buffer, + self.query_start_buffer, + num_padded_tokens, + ) + + return InklingSconvMetadata( + block_table=common_attn_metadata.block_table_tensor, + slot_mapping=common_attn_metadata.slot_mapping[:num_padded_tokens], + seq_idx=self.seq_idx_buffer[:num_padded_tokens], + query_start=self.query_start_buffer[:num_padded_tokens], + ) + + +class InklingSconvBackend(AttentionBackend): + """Custom dummy backend for the sconv sliding-window cache management.""" + + @staticmethod + def get_name() -> str: + return "INKLING_SCONV_SWA" + + @classmethod + def indexes_kv_by_block_stride(cls) -> bool: + # num_blocks is the outermost dim (HND, see get_kv_cache_shape), so the + # padded conv page is read through a strided view. + return True + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + cache_dtype_str: str = "auto", + ) -> tuple[int, ...]: + # HND, num-blocks-first, head-major: [num_blocks, H, N, D]. + return (num_blocks, num_kv_heads, block_size, head_size) + + @staticmethod + def get_kv_cache_stride_order( + include_num_layers_dimension: bool = False, + ) -> tuple[int, ...]: + # Identity: physical layout == logical [num_blocks, H, N, D]. + if include_num_layers_dimension: + return (0, 1, 2, 3, 4) + return (0, 1, 2, 3) + + @staticmethod + def get_impl_cls(): + raise NotImplementedError( + "InklingSconvBackend has no attention impl; the conv runs out-of-band." + ) + + @staticmethod + def get_builder_cls() -> type[InklingSconvMetadataBuilder]: + return InklingSconvMetadataBuilder + + +class InklingConvState(nn.Module, AttentionLayerBase): + """Per-decoder-layer owner emitting one sliding-window conv-state spec.""" + + def __init__( + self, + *, + num_kv_heads: int, + head_dim: int, + hidden_size: int, + kernel_size: int, + prefix: str, + ) -> None: + super().__init__() + self.prefix = prefix + # Bound to the manager-allocated paged cache by bind_kv_cache; a + # placeholder until then. Read out-of-band by InklingShortConv. + self.kv_cache = torch.tensor([]) + tp_size = get_tensor_model_parallel_world_size() + # Guardrails for the conv-state layout below; only these are exercised. + # tp_size <= num_kv_heads keeps >=1 whole KV head per rank (no + # replication/clamping), so the per-head width stays TP-invariant. + assert tp_size <= num_kv_heads, ( + f"sconv SWA cache supports tp_size <= num_kv_heads ({num_kv_heads}), " + f"got {tp_size}" + ) + # Per-rank head count; D is TP-invariant (K/V heads and the hidden + # chunk both scale 1/TP together). The attn-/mlp-output sconv streams + # are hidden-sharded: each rank owns its H/tp chunk (the sublayer + # outputs are reduce-scattered / all-gathered around the conv). + self.num_kv_heads = num_kv_heads // tp_size + hidden_per_head = hidden_size // num_kv_heads + # Packed per-head width: K + V + attn-output chunk + mlp-output chunk, + # padded to a power of two so every layer's conv page is the same size + # and an exact multiple of the attention page (the page unifier then + # scales attention block sizes instead of padding). + raw_head_size = 2 * head_dim + 2 * hidden_per_head + self.head_size = 1 << (raw_head_size - 1).bit_length() + self.sliding_window = kernel_size + self.block_size = kernel_size + # Per-head D-sub-range (offset, width) for each stream. Streams share + # the cache; each writes/reads its own width across all H heads. + self.stream_ranges: tuple[tuple[int, int], ...] = ( + (0, head_dim), # _K + (head_dim, head_dim), # _V + (2 * head_dim, hidden_per_head), # _ATTN + (2 * head_dim + hidden_per_head, hidden_per_head), # _MLP + ) + vllm_config = get_current_vllm_config() + self._dtype = vllm_config.model_config.dtype + assert self._dtype == torch.bfloat16, ( + f"sconv SWA cache supports bfloat16 only, got {self._dtype}" + ) + # Register in the forward context so the runner enumerates this owner as + # an attention-like layer (get_kv_cache_spec / get_attn_backend). + compilation_config = vllm_config.compilation_config + if prefix in compilation_config.static_forward_context: + raise ValueError(f"Duplicate layer name: {prefix}") + compilation_config.static_forward_context[prefix] = self + + def forward(self): ... + + @property + def cache_block_size(self) -> int: + """Return the block size used by cache metadata and physical indexing.""" + if self.kv_cache.numel() > 0: + return self.kv_cache.shape[2] + return self.block_size + + def get_attn_backend(self) -> type[AttentionBackend]: + return InklingSconvBackend + + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + return SlidingWindowSpec( + block_size=self.block_size, + num_kv_heads=self.num_kv_heads, + head_size=self.head_size, + head_size_v=0, # all 4 streams packed into head_size + dtype=self._dtype, + sliding_window=self.sliding_window, + ) diff --git a/vllm/models/inkling/amd/short_conv.py b/vllm/models/inkling/amd/short_conv.py new file mode 100644 index 0000000000..3a4db8f883 --- /dev/null +++ b/vllm/models/inkling/amd/short_conv.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling short convolution: depthwise causal conv1d (+ residual) over a paged +sliding-window conv-state cache. + +Each decoder layer owns one ``InklingConvState`` (``sconv_swa_attn.py``) holding the +manager-allocated paged cache for the layer's 4 sconv streams (K, V, attn-output, +mlp-output), packed head-major into one block. Each ``InklingShortConv`` is a +stateless weight + kernel launcher that, per forward (positions-addressed, the +same path for prefill / decode / mixed), inserts the current tokens' inputs +into their paged slot and convolves each token against the ``W`` taps ending +at its absolute position, reading pre-forward window positions out of the +paged cache via the block table. + +Per-forward metadata (``block_table`` / ``slot_mapping`` / ``seq_idx`` / +``query_start``) is built once by ``InklingSconvMetadataBuilder`` and published under +the owner's prefix in the forward context; the absolute ``positions`` are +threaded in from the model. The insert + conv run in a single ``fused_sconv`` +launch (same path for prefill / decode / mixed / spec). All inputs are +fixed-address persistent buffers and the grid is fixed, so the conv replays +correctly under eager, PIECEWISE, and FULL cudagraphs. +""" + +from __future__ import annotations + +import torch +from torch import nn +from torch.nn.parameter import Parameter + +from vllm.distributed import get_tensor_model_parallel_rank +from vllm.forward_context import get_forward_context +from vllm.model_executor.utils import set_weight_attrs + +from .ops import fused_sconv +from .sconv_swa_attn import InklingConvState, InklingSconvMetadata + + +class InklingShortConv(nn.Module): + def __init__( + self, dim: int, kernel_size: int, owner: InklingConvState, stream_idx: int + ) -> None: + super().__init__() + self.dim = dim + self.kernel_size = kernel_size + self.owner = owner + self.stream_idx = stream_idx + self.tp_rank = get_tensor_model_parallel_rank() + + # Depthwise conv weight; checkpoint stores (dim, 1, W). + self.weight = Parameter(torch.empty(dim, 1, kernel_size), requires_grad=False) + set_weight_attrs(self.weight, {"weight_loader": self.weight_loader}) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None: + if loaded_weight.shape[0] != param.shape[0]: + shard = param.shape[0] + loaded_weight = loaded_weight.narrow(0, self.tp_rank * shard, shard) + param.data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + # x: (num_tokens, dim); positions: (num_tokens,) absolute positions. + attn_metadata = get_forward_context().attn_metadata + if not isinstance(attn_metadata, dict): + # Memory-profiling / no metadata: identity (residual). + return x + m = attn_metadata.get(self.owner.prefix) + if m is None: + return x + assert isinstance(m, InklingSconvMetadata) + cache = self.owner.kv_cache + if cache.numel() == 0: + # Cache not yet bound (profiling before KV alloc): identity. + return x + + off_s, ws = self.owner.stream_ranges[self.stream_idx] + # The hybrid KV-cache planner can enlarge the logical conv block so + # that its physical page size matches the attention caches (for + # example, W=4 becomes 32 when --block-size=128). Metadata slot + # mappings are built with that enlarged size, so index the bound cache + # with its runtime token dimension rather than the kernel window size. + block_size = self.owner.cache_block_size + x = x.contiguous() + weight = self.weight.squeeze(1) # (dim, W) + + return fused_sconv( + x, + weight, + cache, + positions, + m.block_table, + m.seq_idx, + m.slot_mapping, + m.query_start, + off_s, + ws, + block_size, + activation=None, + use_residual=True, + ) diff --git a/vllm/triton_utils/__init__.py b/vllm/triton_utils/__init__.py index e20cb002cd..5c9006c05c 100644 --- a/vllm/triton_utils/__init__.py +++ b/vllm/triton_utils/__init__.py @@ -12,10 +12,16 @@ if TYPE_CHECKING or HAS_TRITON: import triton import triton.language as tl import triton.language.extra.libdevice as tldevice + from triton.experimental import gluon + from triton.experimental.gluon import language as gl + from triton.language.core import _aggregate as aggregate # noqa: E501 else: triton = TritonPlaceholder() tl = TritonLanguagePlaceholder() tldevice = TritonLanguagePlaceholder() + gluon = TritonLanguagePlaceholder() + gl = TritonLanguagePlaceholder() + aggregate = TritonLanguagePlaceholder() from vllm.triton_utils.tensor_descriptor import use_tensor_descriptor @@ -29,5 +35,8 @@ __all__ = [ "tldevice", "LOG2E", "LOGE2", + "gluon", + "gl", + "aggregate", "use_tensor_descriptor", ]