[ROCm] [Model] Enable TML inkling (#48841)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def require_rocm_cdna4() -> None:
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("requires ROCm")
|
||||
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not on_gfx950():
|
||||
pytest.skip("requires CDNA4")
|
||||
@@ -0,0 +1,59 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm guards for shared Inkling model-definition contracts."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.models.inkling as inkling
|
||||
from vllm.lora.utils import get_supported_lora_modules
|
||||
from vllm.model_executor.models.interfaces import supports_lora
|
||||
from vllm.models.inkling.amd import model as amd_model
|
||||
from vllm.models.inkling.amd.mtp import InklingMTP
|
||||
|
||||
|
||||
def test_backend_neutral_definitions_match_nvidia_reference() -> None:
|
||||
inkling_dir = Path(amd_model.__file__).parents[1]
|
||||
for filename in ("logits_processor.py", "mtp.py"):
|
||||
assert (inkling_dir / "amd" / filename).read_bytes() == (
|
||||
inkling_dir / "nvidia" / filename
|
||||
).read_bytes()
|
||||
|
||||
|
||||
def test_rocm_exports_its_aligned_mtp_implementation() -> None:
|
||||
assert inkling.InklingMTP is InklingMTP
|
||||
|
||||
|
||||
def test_rocm_model_exposes_the_upstream_lora_contract() -> None:
|
||||
model_cls = amd_model._TmlForCausalLMBase
|
||||
|
||||
assert supports_lora(model_cls)
|
||||
assert model_cls.packed_modules_mapping == {
|
||||
"qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"],
|
||||
"w13": ["w1", "w3"],
|
||||
}
|
||||
assert model_cls.embedding_modules == {"lm_head": "output_embeddings"}
|
||||
|
||||
model = torch.nn.Module()
|
||||
model.embedding_modules = model_cls.embedding_modules
|
||||
supported = get_supported_lora_modules(model)
|
||||
assert "embed_tokens" not in supported
|
||||
assert "lm_head" in supported
|
||||
|
||||
|
||||
def test_lightseek_bundled_adapter_weights_remain_opt_in() -> None:
|
||||
mapper = amd_model._TmlForCausalLMBase.hf_to_vllm_mapper
|
||||
weight = torch.empty(1)
|
||||
name, mapped_weight = next(
|
||||
iter(
|
||||
mapper.apply([("language_model.layers.3.attn.wq_du.lora_A.weight", weight)])
|
||||
)
|
||||
)
|
||||
|
||||
assert name == "model.layers.3.attn.qkvr.lora_A.weight"
|
||||
assert mapped_weight.shard_id == 0
|
||||
assert amd_model._is_peft_adapter_weight(name)
|
||||
|
||||
lm_head_name = mapper.apply_list(["language_model.lm_head.lora_B.weight"])
|
||||
assert lm_head_name == ["lm_head.lora_B.weight"]
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for Quark OCP MXFP4 weights loaded into AITER MoE layouts."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.inkling.amd.moe import InklingMoE
|
||||
|
||||
|
||||
def _fake_moe() -> tuple[InklingMoE, SimpleNamespace]:
|
||||
moe = InklingMoE.__new__(InklingMoE)
|
||||
nn.Module.__init__(moe)
|
||||
moe.n_routed_experts = 4
|
||||
routed = SimpleNamespace(
|
||||
moe_config=SimpleNamespace(
|
||||
moe_parallel_config=SimpleNamespace(tp_rank=1, tp_size=2)
|
||||
),
|
||||
# Logical intermediate per rank is 3; AITER pads it to 4.
|
||||
w13_weight=nn.Parameter(torch.zeros(4, 8, 2, dtype=torch.uint8), False),
|
||||
w2_weight=nn.Parameter(torch.zeros(4, 5, 3, dtype=torch.uint8), False),
|
||||
w13_weight_scale=nn.Parameter(torch.ones(4, 8, 2, dtype=torch.uint8), False),
|
||||
w2_weight_scale=nn.Parameter(torch.ones(4, 5, 3, dtype=torch.uint8), False),
|
||||
)
|
||||
moe.experts = SimpleNamespace(routed_experts=routed)
|
||||
moe._local_expert_slots = ( # type: ignore[method-assign]
|
||||
lambda: dict(enumerate(range(4)))
|
||||
)
|
||||
return moe, routed
|
||||
|
||||
|
||||
def test_loads_logical_tp_shards_and_preserves_aiter_padding():
|
||||
moe, routed = _fake_moe()
|
||||
|
||||
w13 = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4, 12, 2)
|
||||
w2 = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4, 5, 4)
|
||||
moe.load_expert_weight("experts.w13_weight", w13)
|
||||
moe.load_expert_weight("experts.w2_weight", w2)
|
||||
|
||||
# TP rank 1 consumes the second six interleaved gate/up rows. AITER keeps
|
||||
# one padding row in each destination half and one padding w2 column.
|
||||
torch.testing.assert_close(routed.w13_weight[:, :3], w13[:, 6:12:2])
|
||||
torch.testing.assert_close(routed.w13_weight[:, 4:7], w13[:, 7:12:2])
|
||||
assert torch.count_nonzero(routed.w13_weight[:, 3]) == 0
|
||||
assert torch.count_nonzero(routed.w13_weight[:, 7]) == 0
|
||||
torch.testing.assert_close(routed.w2_weight[:, :, :2], w2[:, :, 2:4])
|
||||
assert torch.count_nonzero(routed.w2_weight[:, :, 2]) == 0
|
||||
|
||||
|
||||
def test_unflattens_quark_scales_and_preserves_scale_padding():
|
||||
moe, routed = _fake_moe()
|
||||
|
||||
flat_w13_scale = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4 * 12, 2)
|
||||
flat_w2_scale = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4 * 5, 4)
|
||||
moe.load_expert_weight("experts.w13_weight_scale", flat_w13_scale)
|
||||
moe.load_expert_weight("experts.w2_weight_scale", flat_w2_scale)
|
||||
|
||||
w13_scale = flat_w13_scale.view(4, 12, 2)
|
||||
w2_scale = flat_w2_scale.view(4, 5, 4)
|
||||
torch.testing.assert_close(routed.w13_weight_scale[:, :3], w13_scale[:, 6:12:2])
|
||||
torch.testing.assert_close(routed.w13_weight_scale[:, 4:7], w13_scale[:, 7:12:2])
|
||||
assert torch.all(routed.w13_weight_scale[:, 3] == 1)
|
||||
assert torch.all(routed.w13_weight_scale[:, 7] == 1)
|
||||
torch.testing.assert_close(routed.w2_weight_scale[:, :, :2], w2_scale[:, :, 2:4])
|
||||
assert torch.all(routed.w2_weight_scale[:, :, 2] == 1)
|
||||
@@ -0,0 +1,425 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for Inkling's ROCm relative-bias paged attention."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.inkling.amd.ops.fa4_rel_attention import (
|
||||
bucket_max_seqlen_q,
|
||||
inkling_fa4_rel_attention,
|
||||
use_gfx950_gluon_decode,
|
||||
use_gfx950_gluon_extend,
|
||||
)
|
||||
from vllm.models.inkling.amd.ops.rel_attention_decode import (
|
||||
decode_split_count,
|
||||
use_split_kv_decode,
|
||||
)
|
||||
|
||||
HEAD_DIM = 128
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def _reference(
|
||||
q: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
rel_logits: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
q_lens: list[int],
|
||||
kv_lens: list[int],
|
||||
rel_extent: int,
|
||||
window_left: int | None,
|
||||
) -> torch.Tensor:
|
||||
num_heads = q.shape[1]
|
||||
num_kv_heads = key_cache.shape[2]
|
||||
gqa_group = num_heads // num_kv_heads
|
||||
out: list[torch.Tensor] = []
|
||||
q_start = 0
|
||||
|
||||
for req, (q_len, kv_len) in enumerate(zip(q_lens, kv_lens)):
|
||||
page_size = key_cache.shape[1]
|
||||
num_pages = (kv_len + page_size - 1) // page_size
|
||||
page_ids = block_table[req, :num_pages].long()
|
||||
k = key_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
|
||||
v = value_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
|
||||
k = k.float().repeat_interleave(gqa_group, dim=1)
|
||||
v = v.float().repeat_interleave(gqa_group, dim=1)
|
||||
|
||||
q_req = q[q_start : q_start + q_len].float()
|
||||
rel_req = rel_logits[q_start : q_start + q_len].float()
|
||||
scores = torch.einsum("qhd,khd->hqk", q_req, k) / HEAD_DIM
|
||||
|
||||
q_pos = torch.arange(q_len, device=q.device)[:, None] + kv_len - q_len
|
||||
k_pos = torch.arange(kv_len, device=q.device)[None, :]
|
||||
distance = q_pos - k_pos
|
||||
rel_idx = distance.clamp(0, rel_extent - 1)
|
||||
bias = rel_req.permute(1, 0, 2).gather(
|
||||
2, rel_idx[None].expand(num_heads, -1, -1)
|
||||
)
|
||||
in_rel_extent = (distance >= 0) & (distance < rel_extent)
|
||||
scores += torch.where(in_rel_extent[None], bias, 0.0)
|
||||
|
||||
masked = distance < 0
|
||||
if window_left is not None:
|
||||
masked |= distance > window_left
|
||||
scores.masked_fill_(masked[None], float("-inf"))
|
||||
out.append(torch.einsum("hqk,khd->qhd", scores.softmax(-1), v))
|
||||
q_start += q_len
|
||||
|
||||
return torch.cat(out).to(DTYPE)
|
||||
|
||||
|
||||
def test_query_length_bucket():
|
||||
assert bucket_max_seqlen_q(1) == 1
|
||||
assert bucket_max_seqlen_q(17) == 32
|
||||
assert bucket_max_seqlen_q(33) == 64
|
||||
|
||||
|
||||
def test_split_kv_dispatch_thresholds():
|
||||
assert not use_split_kv_decode(
|
||||
max_query_len=4,
|
||||
max_kv_len=65536,
|
||||
page_size=128,
|
||||
window_left=-1,
|
||||
)
|
||||
assert not use_split_kv_decode(
|
||||
max_query_len=1,
|
||||
max_kv_len=4096,
|
||||
page_size=16,
|
||||
window_left=-1,
|
||||
)
|
||||
assert use_split_kv_decode(
|
||||
max_query_len=1,
|
||||
max_kv_len=8192,
|
||||
page_size=16,
|
||||
window_left=-1,
|
||||
)
|
||||
assert use_split_kv_decode(
|
||||
max_query_len=1,
|
||||
max_kv_len=512,
|
||||
page_size=128,
|
||||
window_left=511,
|
||||
)
|
||||
assert decode_split_count(512, 511) == 4
|
||||
assert decode_split_count(65536, -1) == 32
|
||||
|
||||
|
||||
def test_gfx950_gluon_dispatch_is_strict(monkeypatch: pytest.MonkeyPatch):
|
||||
import vllm.models.inkling.amd.ops.fa4_rel_attention as rel_attention
|
||||
|
||||
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
|
||||
assert use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=16, head_dim=128)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=4, page_size=128, head_dim=128)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=256)
|
||||
|
||||
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: False)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
|
||||
|
||||
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
|
||||
assert use_gfx950_gluon_extend(
|
||||
max_query_len=4,
|
||||
max_kv_len=8192,
|
||||
page_size=128,
|
||||
head_dim=128,
|
||||
window_left=-1,
|
||||
)
|
||||
assert not use_gfx950_gluon_extend(
|
||||
max_query_len=4,
|
||||
max_kv_len=8192,
|
||||
page_size=128,
|
||||
head_dim=128,
|
||||
window_left=511,
|
||||
)
|
||||
assert not use_gfx950_gluon_extend(
|
||||
max_query_len=4,
|
||||
max_kv_len=8192,
|
||||
page_size=16,
|
||||
head_dim=128,
|
||||
window_left=-1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("page_size", "window_left", "max_kv_len"),
|
||||
[
|
||||
(16, None, None),
|
||||
(16, 15, None),
|
||||
(128, None, 8192),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_ragged_multi_page_relative_attention(
|
||||
page_size: int,
|
||||
window_left: int | None,
|
||||
max_kv_len: int | None,
|
||||
):
|
||||
"""Covers full/chunked prefill, decode, GQA, and the local window."""
|
||||
torch.manual_seed(19 + int(window_left is not None))
|
||||
device = "cuda"
|
||||
q_lens = [17, 1]
|
||||
kv_lens = [35, 80]
|
||||
num_heads = 8
|
||||
num_kv_heads = 2
|
||||
rel_extent = 16
|
||||
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
|
||||
|
||||
q = torch.randn(sum(q_lens), num_heads, HEAD_DIM, device=device)
|
||||
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
|
||||
key_cache = torch.randn(
|
||||
1 + len(q_lens) * max_pages,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
HEAD_DIM,
|
||||
device=device,
|
||||
)
|
||||
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
|
||||
value_cache = torch.randn_like(key_cache)
|
||||
rel_logits = torch.randn(
|
||||
sum(q_lens), num_heads, rel_extent, device=device, dtype=DTYPE
|
||||
)
|
||||
|
||||
block_table = torch.stack(
|
||||
[
|
||||
torch.arange(1 + req * max_pages, 1 + (req + 1) * max_pages)
|
||||
for req in range(len(q_lens))
|
||||
]
|
||||
).to(device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = torch.tensor(
|
||||
[0, *torch.tensor(q_lens).cumsum(0).tolist()],
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
|
||||
preallocated = torch.empty_like(q)
|
||||
actual = inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=bucket_max_seqlen_q(max(q_lens)),
|
||||
softmax_scale=1.0 / HEAD_DIM,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
max_kv_len=max_kv_len,
|
||||
out=preallocated,
|
||||
)
|
||||
assert actual.data_ptr() == preallocated.data_ptr()
|
||||
|
||||
expected = _reference(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
block_table,
|
||||
q_lens,
|
||||
kv_lens,
|
||||
rel_extent,
|
||||
window_left,
|
||||
)
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("page_size", "kv_lens", "num_kv_heads", "rel_extent", "window_left"),
|
||||
[
|
||||
(16, [257, 513], 1, 64, None),
|
||||
(128, [257, 512], 2, 512, 511),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_split_kv_decode_matches_reference(
|
||||
page_size: int,
|
||||
kv_lens: list[int],
|
||||
num_kv_heads: int,
|
||||
rel_extent: int,
|
||||
window_left: int | None,
|
||||
):
|
||||
"""Covers page-16 split-KV and page-128 gfx950 Gluon decode."""
|
||||
torch.manual_seed(41 + page_size)
|
||||
device = "cuda"
|
||||
batch_size = len(kv_lens)
|
||||
q_lens = [1] * batch_size
|
||||
num_heads = 8
|
||||
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
|
||||
|
||||
q = torch.randn(batch_size, num_heads, HEAD_DIM, device=device)
|
||||
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
|
||||
key_cache = torch.randn(
|
||||
batch_size * max_pages,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
HEAD_DIM,
|
||||
device=device,
|
||||
)
|
||||
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
|
||||
value_cache = torch.randn_like(key_cache)
|
||||
rel_logits = torch.randn(
|
||||
batch_size,
|
||||
num_heads,
|
||||
rel_extent,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
block_table = torch.arange(
|
||||
batch_size * max_pages,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
).view(batch_size, max_pages)
|
||||
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = torch.arange(
|
||||
batch_size + 1,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
max_kv_len = 8192 if page_size == 16 else max(kv_lens)
|
||||
|
||||
actual = inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=1,
|
||||
softmax_scale=1.0 / HEAD_DIM,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
max_kv_len=max_kv_len,
|
||||
)
|
||||
expected = _reference(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
block_table,
|
||||
q_lens,
|
||||
kv_lens,
|
||||
rel_extent,
|
||||
window_left,
|
||||
)
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"q_len",
|
||||
"kv_len",
|
||||
"num_kv_heads",
|
||||
"rel_extent",
|
||||
"window_left",
|
||||
"gluon_enabled",
|
||||
),
|
||||
[
|
||||
(1, 513, 2, 512, 511, True),
|
||||
(1, 513, 2, 512, 511, False),
|
||||
(4, 8192, 1, 1024, None, True),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_gfx950_page128_packed_kv_views_match_reference(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
q_len: int,
|
||||
kv_len: int,
|
||||
num_kv_heads: int,
|
||||
rel_extent: int,
|
||||
window_left: int | None,
|
||||
gluon_enabled: bool,
|
||||
):
|
||||
"""Cover both Gluon and split-KV against vLLM's packed KV allocation."""
|
||||
monkeypatch.setenv(
|
||||
"INKLING_GFX950_GLUON",
|
||||
"1" if gluon_enabled else "0",
|
||||
)
|
||||
torch.manual_seed(71 + q_len)
|
||||
device = "cuda"
|
||||
page_size = 128
|
||||
batch_size = 2
|
||||
num_heads = 8
|
||||
q_lens = [q_len] * batch_size
|
||||
kv_lens = [kv_len] * batch_size
|
||||
pages_per_req = (kv_len + page_size - 1) // page_size
|
||||
|
||||
q = torch.randn(
|
||||
batch_size * q_len,
|
||||
num_heads,
|
||||
HEAD_DIM,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
|
||||
|
||||
# FlashAttentionBackend allocates logical [block, head, page, 2 * dim].
|
||||
# Inkling transposes to [block, page, head, 2 * dim] and splits K/V,
|
||||
# producing non-contiguous views whose page/head strides include both.
|
||||
packed_kv = torch.randn(
|
||||
batch_size * pages_per_req,
|
||||
num_kv_heads,
|
||||
page_size,
|
||||
2 * HEAD_DIM,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
key_cache, value_cache = packed_kv.transpose(1, 2).split(HEAD_DIM, dim=-1)
|
||||
assert not key_cache.is_contiguous()
|
||||
assert key_cache.stride() == value_cache.stride()
|
||||
|
||||
rel_logits = torch.randn(
|
||||
batch_size * q_len,
|
||||
num_heads,
|
||||
rel_extent,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
block_table = torch.arange(
|
||||
batch_size * pages_per_req,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
).view(batch_size, pages_per_req)
|
||||
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = torch.arange(
|
||||
0,
|
||||
batch_size * q_len + 1,
|
||||
q_len,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
|
||||
actual = inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=q_len,
|
||||
softmax_scale=1.0 / HEAD_DIM,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
max_kv_len=kv_len,
|
||||
)
|
||||
expected = _reference(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
block_table,
|
||||
q_lens,
|
||||
kv_lens,
|
||||
rel_extent,
|
||||
window_left,
|
||||
)
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
|
||||
@@ -0,0 +1,116 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm numerical-equivalence tests for the fused MTP depth-layer input kernel.
|
||||
|
||||
``embed_dual_rmsnorm_cat`` must match the unfused module sequence numerically:
|
||||
each rmsnorm computes in fp32 and rounds to bf16 at the same points as the
|
||||
vendored ``rmsnorm`` kernel (including the bf16 round-trip between the
|
||||
chained backbone embed_norm and the depth embed_norm), and the fused row
|
||||
gather matches ``F.embedding``.
|
||||
"""
|
||||
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.inkling.amd.ops.norm import (
|
||||
embed_dual_rmsnorm_cat,
|
||||
embed_rmsnorm,
|
||||
rmsnorm,
|
||||
)
|
||||
|
||||
EPS = 1e-6
|
||||
VOCAB = 4096
|
||||
BF16_ATOL = torch.finfo(torch.bfloat16).eps
|
||||
|
||||
|
||||
def _assert_bf16_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
# Different Triton reduction schedules can move a small number of outputs
|
||||
# by one bf16 epsilon while remaining numerically equivalent.
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=BF16_ATOL)
|
||||
|
||||
|
||||
def _ref(hidden, w_h, w_e, emb, w_pre=None):
|
||||
if w_pre is not None:
|
||||
emb = rmsnorm(emb, w_pre, EPS)
|
||||
return torch.cat([rmsnorm(hidden, w_h, EPS), rmsnorm(emb, w_e, EPS)], dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1536, 6144])
|
||||
@pytest.mark.parametrize("t", [0, 1, 7, 256])
|
||||
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
|
||||
def test_embed_dual_rmsnorm_cat(n: int, t: int, ids_dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(0)
|
||||
dev = "cuda"
|
||||
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
|
||||
w_h = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
w_e = (1 + 0.01 * torch.randn(n, device=dev)).to(torch.bfloat16)
|
||||
w_pre = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
|
||||
hidden = (torch.randn(t, n, device=dev) * 2).to(torch.bfloat16)
|
||||
emb = table[ids.long()]
|
||||
|
||||
# Fused gather + chained backbone pre-norm (the decode draft-step path).
|
||||
out = embed_dual_rmsnorm_cat(
|
||||
hidden,
|
||||
w_h,
|
||||
w_e,
|
||||
EPS,
|
||||
input_ids=ids,
|
||||
embed_table=table,
|
||||
pre_norm_weight=w_pre,
|
||||
)
|
||||
assert out.shape == (t, 2 * n)
|
||||
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb, w_pre))
|
||||
|
||||
# Precomputed embeds, no pre-norm (draft prefill with target-merged MM
|
||||
# embeddings, already backbone-normed).
|
||||
out = embed_dual_rmsnorm_cat(hidden, w_h, w_e, EPS, embeds=emb)
|
||||
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
|
||||
|
||||
# Fused gather, no pre-norm (use_embed_norm=False).
|
||||
out = embed_dual_rmsnorm_cat(
|
||||
hidden, w_h, w_e, EPS, input_ids=ids, embed_table=table
|
||||
)
|
||||
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1536, 6144])
|
||||
@pytest.mark.parametrize("t", [0, 1, 7, 256])
|
||||
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
|
||||
def test_embed_rmsnorm(n: int, t: int, ids_dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(0)
|
||||
dev = "cuda"
|
||||
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
|
||||
w = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
|
||||
ref_emb = table[ids.long()]
|
||||
|
||||
# Gather + embed_norm (base model / MTP prefill embed path).
|
||||
out = cast(torch.Tensor, embed_rmsnorm(ids, table, w, EPS))
|
||||
assert out.shape == (t, n)
|
||||
_assert_bf16_close(out, rmsnorm(ref_emb, w, EPS) if t else ref_emb)
|
||||
|
||||
# Pure gather (use_embed_norm=False / replicated module forward).
|
||||
out = cast(torch.Tensor, embed_rmsnorm(ids, table, None, EPS))
|
||||
assert torch.equal(out, ref_emb)
|
||||
|
||||
# Chained first-layer attn_norm (the target text-path forward): one launch
|
||||
# emits both the residual and layer 0's normed attention input.
|
||||
w_chain = (1 + 0.05 * torch.randn(n, device=dev)).to(torch.bfloat16)
|
||||
res, attn_in = cast(
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
embed_rmsnorm(ids, table, w, EPS, chain_weight=w_chain),
|
||||
)
|
||||
ref_res = rmsnorm(ref_emb, w, EPS) if t else ref_emb
|
||||
_assert_bf16_close(res, ref_res)
|
||||
_assert_bf16_close(attn_in, rmsnorm(ref_res, w_chain, EPS) if t else ref_res)
|
||||
|
||||
# Chained without embed_norm (use_embed_norm=False).
|
||||
res, attn_in = cast(
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
embed_rmsnorm(ids, table, None, EPS, chain_weight=w_chain),
|
||||
)
|
||||
assert torch.equal(res, ref_emb)
|
||||
_assert_bf16_close(attn_in, rmsnorm(ref_emb, w_chain, EPS) if t else ref_emb)
|
||||
@@ -0,0 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.inkling.amd.sconv_swa_attn import InklingConvState
|
||||
|
||||
|
||||
def test_runtime_sconv_block_size_tracks_unified_cache_page():
|
||||
"""The cache planner may enlarge W=4 blocks to match attention pages."""
|
||||
owner = InklingConvState.__new__(InklingConvState)
|
||||
nn.Module.__init__(owner)
|
||||
owner.block_size = 4
|
||||
|
||||
owner.kv_cache = torch.tensor([])
|
||||
assert owner.cache_block_size == 4
|
||||
|
||||
owner.kv_cache = torch.empty(2, 1, 32, 1024)
|
||||
assert owner.cache_block_size == 32
|
||||
Reference in New Issue
Block a user