[ROCm] [Model] Enable TML inkling (#48841)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
TJian
2026-07-27 10:05:46 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent bc3629b1c4
commit 0906123953
33 changed files with 9213 additions and 9 deletions
+17
View File
@@ -0,0 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.platforms import current_platform
@pytest.fixture(autouse=True)
def require_rocm_cdna4() -> None:
if not current_platform.is_rocm():
pytest.skip("requires ROCm")
from vllm.platforms.rocm import on_gfx950
if not on_gfx950():
pytest.skip("requires CDNA4")
@@ -0,0 +1,59 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm guards for shared Inkling model-definition contracts."""
from pathlib import Path
import torch
import vllm.models.inkling as inkling
from vllm.lora.utils import get_supported_lora_modules
from vllm.model_executor.models.interfaces import supports_lora
from vllm.models.inkling.amd import model as amd_model
from vllm.models.inkling.amd.mtp import InklingMTP
def test_backend_neutral_definitions_match_nvidia_reference() -> None:
inkling_dir = Path(amd_model.__file__).parents[1]
for filename in ("logits_processor.py", "mtp.py"):
assert (inkling_dir / "amd" / filename).read_bytes() == (
inkling_dir / "nvidia" / filename
).read_bytes()
def test_rocm_exports_its_aligned_mtp_implementation() -> None:
assert inkling.InklingMTP is InklingMTP
def test_rocm_model_exposes_the_upstream_lora_contract() -> None:
model_cls = amd_model._TmlForCausalLMBase
assert supports_lora(model_cls)
assert model_cls.packed_modules_mapping == {
"qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"],
"w13": ["w1", "w3"],
}
assert model_cls.embedding_modules == {"lm_head": "output_embeddings"}
model = torch.nn.Module()
model.embedding_modules = model_cls.embedding_modules
supported = get_supported_lora_modules(model)
assert "embed_tokens" not in supported
assert "lm_head" in supported
def test_lightseek_bundled_adapter_weights_remain_opt_in() -> None:
mapper = amd_model._TmlForCausalLMBase.hf_to_vllm_mapper
weight = torch.empty(1)
name, mapped_weight = next(
iter(
mapper.apply([("language_model.layers.3.attn.wq_du.lora_A.weight", weight)])
)
)
assert name == "model.layers.3.attn.qkvr.lora_A.weight"
assert mapped_weight.shard_id == 0
assert amd_model._is_peft_adapter_weight(name)
lm_head_name = mapper.apply_list(["language_model.lm_head.lora_B.weight"])
assert lm_head_name == ["lm_head.lora_B.weight"]
@@ -0,0 +1,67 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for Quark OCP MXFP4 weights loaded into AITER MoE layouts."""
from types import SimpleNamespace
import torch
from torch import nn
from vllm.models.inkling.amd.moe import InklingMoE
def _fake_moe() -> tuple[InklingMoE, SimpleNamespace]:
moe = InklingMoE.__new__(InklingMoE)
nn.Module.__init__(moe)
moe.n_routed_experts = 4
routed = SimpleNamespace(
moe_config=SimpleNamespace(
moe_parallel_config=SimpleNamespace(tp_rank=1, tp_size=2)
),
# Logical intermediate per rank is 3; AITER pads it to 4.
w13_weight=nn.Parameter(torch.zeros(4, 8, 2, dtype=torch.uint8), False),
w2_weight=nn.Parameter(torch.zeros(4, 5, 3, dtype=torch.uint8), False),
w13_weight_scale=nn.Parameter(torch.ones(4, 8, 2, dtype=torch.uint8), False),
w2_weight_scale=nn.Parameter(torch.ones(4, 5, 3, dtype=torch.uint8), False),
)
moe.experts = SimpleNamespace(routed_experts=routed)
moe._local_expert_slots = ( # type: ignore[method-assign]
lambda: dict(enumerate(range(4)))
)
return moe, routed
def test_loads_logical_tp_shards_and_preserves_aiter_padding():
moe, routed = _fake_moe()
w13 = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4, 12, 2)
w2 = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4, 5, 4)
moe.load_expert_weight("experts.w13_weight", w13)
moe.load_expert_weight("experts.w2_weight", w2)
# TP rank 1 consumes the second six interleaved gate/up rows. AITER keeps
# one padding row in each destination half and one padding w2 column.
torch.testing.assert_close(routed.w13_weight[:, :3], w13[:, 6:12:2])
torch.testing.assert_close(routed.w13_weight[:, 4:7], w13[:, 7:12:2])
assert torch.count_nonzero(routed.w13_weight[:, 3]) == 0
assert torch.count_nonzero(routed.w13_weight[:, 7]) == 0
torch.testing.assert_close(routed.w2_weight[:, :, :2], w2[:, :, 2:4])
assert torch.count_nonzero(routed.w2_weight[:, :, 2]) == 0
def test_unflattens_quark_scales_and_preserves_scale_padding():
moe, routed = _fake_moe()
flat_w13_scale = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4 * 12, 2)
flat_w2_scale = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4 * 5, 4)
moe.load_expert_weight("experts.w13_weight_scale", flat_w13_scale)
moe.load_expert_weight("experts.w2_weight_scale", flat_w2_scale)
w13_scale = flat_w13_scale.view(4, 12, 2)
w2_scale = flat_w2_scale.view(4, 5, 4)
torch.testing.assert_close(routed.w13_weight_scale[:, :3], w13_scale[:, 6:12:2])
torch.testing.assert_close(routed.w13_weight_scale[:, 4:7], w13_scale[:, 7:12:2])
assert torch.all(routed.w13_weight_scale[:, 3] == 1)
assert torch.all(routed.w13_weight_scale[:, 7] == 1)
torch.testing.assert_close(routed.w2_weight_scale[:, :, :2], w2_scale[:, :, 2:4])
assert torch.all(routed.w2_weight_scale[:, :, 2] == 1)
@@ -0,0 +1,425 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for Inkling's ROCm relative-bias paged attention."""
import pytest
import torch
from vllm.models.inkling.amd.ops.fa4_rel_attention import (
bucket_max_seqlen_q,
inkling_fa4_rel_attention,
use_gfx950_gluon_decode,
use_gfx950_gluon_extend,
)
from vllm.models.inkling.amd.ops.rel_attention_decode import (
decode_split_count,
use_split_kv_decode,
)
HEAD_DIM = 128
DTYPE = torch.bfloat16
def _reference(
q: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
rel_logits: torch.Tensor,
block_table: torch.Tensor,
q_lens: list[int],
kv_lens: list[int],
rel_extent: int,
window_left: int | None,
) -> torch.Tensor:
num_heads = q.shape[1]
num_kv_heads = key_cache.shape[2]
gqa_group = num_heads // num_kv_heads
out: list[torch.Tensor] = []
q_start = 0
for req, (q_len, kv_len) in enumerate(zip(q_lens, kv_lens)):
page_size = key_cache.shape[1]
num_pages = (kv_len + page_size - 1) // page_size
page_ids = block_table[req, :num_pages].long()
k = key_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
v = value_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
k = k.float().repeat_interleave(gqa_group, dim=1)
v = v.float().repeat_interleave(gqa_group, dim=1)
q_req = q[q_start : q_start + q_len].float()
rel_req = rel_logits[q_start : q_start + q_len].float()
scores = torch.einsum("qhd,khd->hqk", q_req, k) / HEAD_DIM
q_pos = torch.arange(q_len, device=q.device)[:, None] + kv_len - q_len
k_pos = torch.arange(kv_len, device=q.device)[None, :]
distance = q_pos - k_pos
rel_idx = distance.clamp(0, rel_extent - 1)
bias = rel_req.permute(1, 0, 2).gather(
2, rel_idx[None].expand(num_heads, -1, -1)
)
in_rel_extent = (distance >= 0) & (distance < rel_extent)
scores += torch.where(in_rel_extent[None], bias, 0.0)
masked = distance < 0
if window_left is not None:
masked |= distance > window_left
scores.masked_fill_(masked[None], float("-inf"))
out.append(torch.einsum("hqk,khd->qhd", scores.softmax(-1), v))
q_start += q_len
return torch.cat(out).to(DTYPE)
def test_query_length_bucket():
assert bucket_max_seqlen_q(1) == 1
assert bucket_max_seqlen_q(17) == 32
assert bucket_max_seqlen_q(33) == 64
def test_split_kv_dispatch_thresholds():
assert not use_split_kv_decode(
max_query_len=4,
max_kv_len=65536,
page_size=128,
window_left=-1,
)
assert not use_split_kv_decode(
max_query_len=1,
max_kv_len=4096,
page_size=16,
window_left=-1,
)
assert use_split_kv_decode(
max_query_len=1,
max_kv_len=8192,
page_size=16,
window_left=-1,
)
assert use_split_kv_decode(
max_query_len=1,
max_kv_len=512,
page_size=128,
window_left=511,
)
assert decode_split_count(512, 511) == 4
assert decode_split_count(65536, -1) == 32
def test_gfx950_gluon_dispatch_is_strict(monkeypatch: pytest.MonkeyPatch):
import vllm.models.inkling.amd.ops.fa4_rel_attention as rel_attention
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
assert use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=16, head_dim=128)
assert not use_gfx950_gluon_decode(max_query_len=4, page_size=128, head_dim=128)
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=256)
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: False)
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
assert use_gfx950_gluon_extend(
max_query_len=4,
max_kv_len=8192,
page_size=128,
head_dim=128,
window_left=-1,
)
assert not use_gfx950_gluon_extend(
max_query_len=4,
max_kv_len=8192,
page_size=128,
head_dim=128,
window_left=511,
)
assert not use_gfx950_gluon_extend(
max_query_len=4,
max_kv_len=8192,
page_size=16,
head_dim=128,
window_left=-1,
)
@pytest.mark.parametrize(
("page_size", "window_left", "max_kv_len"),
[
(16, None, None),
(16, 15, None),
(128, None, 8192),
],
)
@torch.inference_mode()
def test_ragged_multi_page_relative_attention(
page_size: int,
window_left: int | None,
max_kv_len: int | None,
):
"""Covers full/chunked prefill, decode, GQA, and the local window."""
torch.manual_seed(19 + int(window_left is not None))
device = "cuda"
q_lens = [17, 1]
kv_lens = [35, 80]
num_heads = 8
num_kv_heads = 2
rel_extent = 16
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
q = torch.randn(sum(q_lens), num_heads, HEAD_DIM, device=device)
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
key_cache = torch.randn(
1 + len(q_lens) * max_pages,
page_size,
num_kv_heads,
HEAD_DIM,
device=device,
)
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
value_cache = torch.randn_like(key_cache)
rel_logits = torch.randn(
sum(q_lens), num_heads, rel_extent, device=device, dtype=DTYPE
)
block_table = torch.stack(
[
torch.arange(1 + req * max_pages, 1 + (req + 1) * max_pages)
for req in range(len(q_lens))
]
).to(device=device, dtype=torch.int32)
cu_seqlens_q = torch.tensor(
[0, *torch.tensor(q_lens).cumsum(0).tolist()],
device=device,
dtype=torch.int32,
)
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
window_size = (-1, -1) if window_left is None else (window_left, 0)
preallocated = torch.empty_like(q)
actual = inkling_fa4_rel_attention(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=bucket_max_seqlen_q(max(q_lens)),
softmax_scale=1.0 / HEAD_DIM,
causal=True,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
max_kv_len=max_kv_len,
out=preallocated,
)
assert actual.data_ptr() == preallocated.data_ptr()
expected = _reference(
q,
key_cache,
value_cache,
rel_logits,
block_table,
q_lens,
kv_lens,
rel_extent,
window_left,
)
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
@pytest.mark.parametrize(
("page_size", "kv_lens", "num_kv_heads", "rel_extent", "window_left"),
[
(16, [257, 513], 1, 64, None),
(128, [257, 512], 2, 512, 511),
],
)
@torch.inference_mode()
def test_split_kv_decode_matches_reference(
page_size: int,
kv_lens: list[int],
num_kv_heads: int,
rel_extent: int,
window_left: int | None,
):
"""Covers page-16 split-KV and page-128 gfx950 Gluon decode."""
torch.manual_seed(41 + page_size)
device = "cuda"
batch_size = len(kv_lens)
q_lens = [1] * batch_size
num_heads = 8
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
q = torch.randn(batch_size, num_heads, HEAD_DIM, device=device)
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
key_cache = torch.randn(
batch_size * max_pages,
page_size,
num_kv_heads,
HEAD_DIM,
device=device,
)
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
value_cache = torch.randn_like(key_cache)
rel_logits = torch.randn(
batch_size,
num_heads,
rel_extent,
device=device,
dtype=DTYPE,
)
block_table = torch.arange(
batch_size * max_pages,
device=device,
dtype=torch.int32,
).view(batch_size, max_pages)
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
cu_seqlens_q = torch.arange(
batch_size + 1,
device=device,
dtype=torch.int32,
)
window_size = (-1, -1) if window_left is None else (window_left, 0)
max_kv_len = 8192 if page_size == 16 else max(kv_lens)
actual = inkling_fa4_rel_attention(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=1,
softmax_scale=1.0 / HEAD_DIM,
causal=True,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
max_kv_len=max_kv_len,
)
expected = _reference(
q,
key_cache,
value_cache,
rel_logits,
block_table,
q_lens,
kv_lens,
rel_extent,
window_left,
)
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
@pytest.mark.parametrize(
(
"q_len",
"kv_len",
"num_kv_heads",
"rel_extent",
"window_left",
"gluon_enabled",
),
[
(1, 513, 2, 512, 511, True),
(1, 513, 2, 512, 511, False),
(4, 8192, 1, 1024, None, True),
],
)
@torch.inference_mode()
def test_gfx950_page128_packed_kv_views_match_reference(
monkeypatch: pytest.MonkeyPatch,
q_len: int,
kv_len: int,
num_kv_heads: int,
rel_extent: int,
window_left: int | None,
gluon_enabled: bool,
):
"""Cover both Gluon and split-KV against vLLM's packed KV allocation."""
monkeypatch.setenv(
"INKLING_GFX950_GLUON",
"1" if gluon_enabled else "0",
)
torch.manual_seed(71 + q_len)
device = "cuda"
page_size = 128
batch_size = 2
num_heads = 8
q_lens = [q_len] * batch_size
kv_lens = [kv_len] * batch_size
pages_per_req = (kv_len + page_size - 1) // page_size
q = torch.randn(
batch_size * q_len,
num_heads,
HEAD_DIM,
device=device,
dtype=DTYPE,
)
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
# FlashAttentionBackend allocates logical [block, head, page, 2 * dim].
# Inkling transposes to [block, page, head, 2 * dim] and splits K/V,
# producing non-contiguous views whose page/head strides include both.
packed_kv = torch.randn(
batch_size * pages_per_req,
num_kv_heads,
page_size,
2 * HEAD_DIM,
device=device,
dtype=DTYPE,
)
key_cache, value_cache = packed_kv.transpose(1, 2).split(HEAD_DIM, dim=-1)
assert not key_cache.is_contiguous()
assert key_cache.stride() == value_cache.stride()
rel_logits = torch.randn(
batch_size * q_len,
num_heads,
rel_extent,
device=device,
dtype=DTYPE,
)
block_table = torch.arange(
batch_size * pages_per_req,
device=device,
dtype=torch.int32,
).view(batch_size, pages_per_req)
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
cu_seqlens_q = torch.arange(
0,
batch_size * q_len + 1,
q_len,
device=device,
dtype=torch.int32,
)
window_size = (-1, -1) if window_left is None else (window_left, 0)
actual = inkling_fa4_rel_attention(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=q_len,
softmax_scale=1.0 / HEAD_DIM,
causal=True,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
max_kv_len=kv_len,
)
expected = _reference(
q,
key_cache,
value_cache,
rel_logits,
block_table,
q_lens,
kv_lens,
rel_extent,
window_left,
)
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm numerical-equivalence tests for the fused MTP depth-layer input kernel.
``embed_dual_rmsnorm_cat`` must match the unfused module sequence numerically:
each rmsnorm computes in fp32 and rounds to bf16 at the same points as the
vendored ``rmsnorm`` kernel (including the bf16 round-trip between the
chained backbone embed_norm and the depth embed_norm), and the fused row
gather matches ``F.embedding``.
"""
from typing import cast
import pytest
import torch
from vllm.models.inkling.amd.ops.norm import (
embed_dual_rmsnorm_cat,
embed_rmsnorm,
rmsnorm,
)
EPS = 1e-6
VOCAB = 4096
BF16_ATOL = torch.finfo(torch.bfloat16).eps
def _assert_bf16_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
# Different Triton reduction schedules can move a small number of outputs
# by one bf16 epsilon while remaining numerically equivalent.
torch.testing.assert_close(actual, expected, rtol=0, atol=BF16_ATOL)
def _ref(hidden, w_h, w_e, emb, w_pre=None):
if w_pre is not None:
emb = rmsnorm(emb, w_pre, EPS)
return torch.cat([rmsnorm(hidden, w_h, EPS), rmsnorm(emb, w_e, EPS)], dim=-1)
@pytest.mark.parametrize("n", [1536, 6144])
@pytest.mark.parametrize("t", [0, 1, 7, 256])
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
def test_embed_dual_rmsnorm_cat(n: int, t: int, ids_dtype: torch.dtype) -> None:
torch.manual_seed(0)
dev = "cuda"
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
w_h = torch.randn(n, device=dev).to(torch.bfloat16)
w_e = (1 + 0.01 * torch.randn(n, device=dev)).to(torch.bfloat16)
w_pre = torch.randn(n, device=dev).to(torch.bfloat16)
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
hidden = (torch.randn(t, n, device=dev) * 2).to(torch.bfloat16)
emb = table[ids.long()]
# Fused gather + chained backbone pre-norm (the decode draft-step path).
out = embed_dual_rmsnorm_cat(
hidden,
w_h,
w_e,
EPS,
input_ids=ids,
embed_table=table,
pre_norm_weight=w_pre,
)
assert out.shape == (t, 2 * n)
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb, w_pre))
# Precomputed embeds, no pre-norm (draft prefill with target-merged MM
# embeddings, already backbone-normed).
out = embed_dual_rmsnorm_cat(hidden, w_h, w_e, EPS, embeds=emb)
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
# Fused gather, no pre-norm (use_embed_norm=False).
out = embed_dual_rmsnorm_cat(
hidden, w_h, w_e, EPS, input_ids=ids, embed_table=table
)
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
@pytest.mark.parametrize("n", [1536, 6144])
@pytest.mark.parametrize("t", [0, 1, 7, 256])
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
def test_embed_rmsnorm(n: int, t: int, ids_dtype: torch.dtype) -> None:
torch.manual_seed(0)
dev = "cuda"
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
w = torch.randn(n, device=dev).to(torch.bfloat16)
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
ref_emb = table[ids.long()]
# Gather + embed_norm (base model / MTP prefill embed path).
out = cast(torch.Tensor, embed_rmsnorm(ids, table, w, EPS))
assert out.shape == (t, n)
_assert_bf16_close(out, rmsnorm(ref_emb, w, EPS) if t else ref_emb)
# Pure gather (use_embed_norm=False / replicated module forward).
out = cast(torch.Tensor, embed_rmsnorm(ids, table, None, EPS))
assert torch.equal(out, ref_emb)
# Chained first-layer attn_norm (the target text-path forward): one launch
# emits both the residual and layer 0's normed attention input.
w_chain = (1 + 0.05 * torch.randn(n, device=dev)).to(torch.bfloat16)
res, attn_in = cast(
tuple[torch.Tensor, torch.Tensor],
embed_rmsnorm(ids, table, w, EPS, chain_weight=w_chain),
)
ref_res = rmsnorm(ref_emb, w, EPS) if t else ref_emb
_assert_bf16_close(res, ref_res)
_assert_bf16_close(attn_in, rmsnorm(ref_res, w_chain, EPS) if t else ref_res)
# Chained without embed_norm (use_embed_norm=False).
res, attn_in = cast(
tuple[torch.Tensor, torch.Tensor],
embed_rmsnorm(ids, table, None, EPS, chain_weight=w_chain),
)
assert torch.equal(res, ref_emb)
_assert_bf16_close(attn_in, rmsnorm(ref_emb, w_chain, EPS) if t else ref_emb)
@@ -0,0 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import nn
from vllm.models.inkling.amd.sconv_swa_attn import InklingConvState
def test_runtime_sconv_block_size_tracks_unified_cache_page():
"""The cache planner may enlarge W=4 blocks to match attention pages."""
owner = InklingConvState.__new__(InklingConvState)
nn.Module.__init__(owner)
owner.block_size = 4
owner.kv_cache = torch.tensor([])
assert owner.cache_block_size == 4
owner.kv_cache = torch.empty(2, 1, 32, 1024)
assert owner.cache_block_size == 32