[ROCm][MLA] Add an AITER ASM round-robin decode route for DCP multi-token verify (#56861)
Signed-off-by: Xiaohu Guo <Xiaohu.Guo@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: seungrokj <144636725+seungrokj@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Claude Opus 5
seungrokj
Cursor
parent
5ba2e3090b
commit
25624f65cf
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Routing/selection tests for the AITER ASM cprr DCP-verify route.
|
||||
|
||||
Companion to the upstream segmented-verify tests. Everything here is pure CPU:
|
||||
it pins the *decisions* (which route a KV group takes, which head count the
|
||||
kernel runs at, which configurations are refused) rather than kernel numerics.
|
||||
The numerics parity test against the segmented route needs a GPU and both
|
||||
routes built; it is marked and skipped when unavailable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
rocm_aiter_mla = pytest.importorskip(
|
||||
"vllm.v1.attention.backends.mla.rocm_aiter_mla",
|
||||
reason="ROCm AITER MLA backend not available",
|
||||
)
|
||||
|
||||
_heads = rocm_aiter_mla._asm_dcp_verify_heads
|
||||
_configured = rocm_aiter_mla._asm_dcp_verify_configured
|
||||
_select_route = rocm_aiter_mla._select_dcp_decode_route
|
||||
Route = rocm_aiter_mla._DCPDecodeRoute
|
||||
NATIVE = rocm_aiter_mla._NATIVE_CPRR_HEADS
|
||||
MIN_QLEN = rocm_aiter_mla._MIN_CPRR_QLEN
|
||||
|
||||
|
||||
def test_dcp_verify_env_defaults_to_segmented(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", raising=False)
|
||||
assert rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "segmented"
|
||||
|
||||
|
||||
def test_dcp_verify_env_rejects_unknown_route(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "unknown")
|
||||
with pytest.raises(ValueError, match="Invalid value"):
|
||||
_ = rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# _asm_dcp_verify_heads -- which head count the cprr kernel runs at
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", NATIVE)
|
||||
def test_native_head_counts_are_used_as_is(n):
|
||||
assert _heads(n) == n
|
||||
|
||||
|
||||
def test_non_native_head_count_pads_up_to_the_next_native():
|
||||
"""K3 at TP8/DCP8 gathers 96 heads for the target; the kernel has no 96
|
||||
variant, so it must run at 128, not silently fall out of the asm path."""
|
||||
assert _heads(96) == 128
|
||||
assert _heads(1) == 16
|
||||
assert _heads(17) == 32
|
||||
assert _heads(65) == 128
|
||||
|
||||
|
||||
def test_head_count_above_the_largest_native_is_unservable():
|
||||
"""0 means 'cannot serve'; anything else would route a shape the kernel
|
||||
has no build for."""
|
||||
assert _heads(max(NATIVE) + 1) == 0
|
||||
assert _heads(256) == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# _asm_dcp_verify_configured -- reachability
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def on_gfx950(monkeypatch):
|
||||
"""The cprr kernels are built only for gfx950, so the gate checks the arch.
|
||||
Patched rather than detected, so both branches run on any runner."""
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
|
||||
|
||||
|
||||
def test_route_needs_dcp(on_gfx950):
|
||||
assert _configured(dcp_world_size=1, cp_interleave=1, multi_token_decode=True) is (
|
||||
False
|
||||
)
|
||||
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is (
|
||||
True
|
||||
)
|
||||
|
||||
|
||||
def test_round_robin_interleave_other_than_one_is_excluded(on_gfx950):
|
||||
"""The kernel's global-position causal window assumes interleave 1, the
|
||||
same restriction the segmented gate carries."""
|
||||
assert _configured(dcp_world_size=8, cp_interleave=2, multi_token_decode=True) is (
|
||||
False
|
||||
)
|
||||
assert _configured(dcp_world_size=8, cp_interleave=16, multi_token_decode=True) is (
|
||||
False
|
||||
)
|
||||
|
||||
|
||||
def test_route_needs_multi_token_decode(on_gfx950):
|
||||
"""Single-token decode is already served. Enabling the route for it would
|
||||
pad the head count on every step and buy nothing."""
|
||||
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=False) is (
|
||||
False
|
||||
)
|
||||
|
||||
|
||||
def test_route_needs_gfx950(monkeypatch):
|
||||
"""hsa/gfx942/mla and hsa/gfx1250/mla carry no cprr rows, so off gfx950 the
|
||||
kernel lookup fails at the first verify step rather than falling back."""
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx950", lambda: False)
|
||||
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is (
|
||||
False
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# qlen floor
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_min_cprr_qlen_is_above_two():
|
||||
"""Qlen 2 is below CPRR's floor and must use segmented verification."""
|
||||
assert MIN_QLEN > 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"supports,causal,qlen,asm,expected",
|
||||
[
|
||||
(True, True, 1, True, Route.PLAIN),
|
||||
(True, True, 2, True, Route.SEGMENTED),
|
||||
(True, True, 3, True, Route.CPRR),
|
||||
(True, True, 4, True, Route.CPRR),
|
||||
(True, True, 4, False, Route.SEGMENTED),
|
||||
(True, False, 2, True, Route.PLAIN),
|
||||
(True, False, 4, False, Route.PLAIN),
|
||||
(False, True, 3, True, Route.CPRR),
|
||||
],
|
||||
)
|
||||
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
|
||||
assert _select_route(supports, causal, qlen, asm) is expected
|
||||
|
||||
|
||||
def test_causal_multi_token_batch_without_a_valid_route_fails():
|
||||
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
|
||||
_select_route(False, True, 2, True)
|
||||
|
||||
|
||||
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
|
||||
# the real builder and the real asm kernel on a gfx950 GPU.
|
||||
@@ -0,0 +1,540 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GPU numerics for the AITER ASM cprr DCP-verify route.
|
||||
|
||||
Runs the REAL metadata builder (``AiterMLAMetadataBuilder.build``, so
|
||||
``get_mla_metadata_v1`` is exercised with the cprr kwargs and the persistent
|
||||
schedule buffers are produced) and the REAL asm kernel, on ONE GPU with the DCP
|
||||
ranks simulated one after another.
|
||||
|
||||
That is legitimate because the aiter MLA backend contains no collective on this
|
||||
path: the query all-gather happens in the *layer*, strictly before
|
||||
``forward_mqa``, so the head fold, the round-robin kernel call and the un-fold
|
||||
are pure per-rank local arithmetic in which ``cp_world_size``/``cp_rank`` are
|
||||
just numbers feeding the causal-mask maths. Restricting a shard to a residue
|
||||
class of global positions is exactly what DCP rank r sees.
|
||||
|
||||
Three properties are checked:
|
||||
* each shard matches an exact torch reference over the positions it holds,
|
||||
with causality applied on the GLOBAL position (the thing cprr reconstructs
|
||||
in-kernel and the segmented route instead carries in per-row lengths);
|
||||
* the LSE merge of all shards reproduces full-context attention;
|
||||
* a shard told the wrong ``cp_rank`` fails both -- without this positive
|
||||
control a passing run proves nothing, since a kernel that ignored global
|
||||
positions still produces plausible-looking output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found # noqa: E402
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402
|
||||
|
||||
KV_LORA_RANK = 512
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5
|
||||
|
||||
DCP = 8
|
||||
NUM_REQS = 8
|
||||
CTX = 256
|
||||
QLEN = 5 # 1 + num_speculative_tokens; must clear _MIN_CPRR_QLEN
|
||||
|
||||
# fp8-e4m3 keeps 3 mantissa bits, so the relative error floor is ~6.2e-2. Gates
|
||||
# are relative for the same reason: an absolute bound would just be a bound on
|
||||
# the output magnitude.
|
||||
SHARD_RTOL = 1.0e-1
|
||||
MERGE_RTOL = 5.0e-2
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-R1"
|
||||
|
||||
|
||||
def _gpu_available() -> bool:
|
||||
return is_aiter_found() and torch.accelerator.is_available()
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not is_aiter_found():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
requires_cprr = pytest.mark.skipif(
|
||||
not (_gpu_available() and _on_gfx950()),
|
||||
reason="cprr DCP verify needs ROCm + AITER on gfx950",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def select_asm_route(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "asm")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# reference maths
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _torch_reference(q_f32, kv_f32, ctx, qlen, scale, positions):
|
||||
"""Exact absorbed-MLA decode over an explicit set of GLOBAL KV positions.
|
||||
|
||||
A shard sees only ``positions``, but causality still uses the global
|
||||
position: query token i (global position ctx + i) may attend to any p in
|
||||
positions with p <= ctx + i. Restricting ``positions`` to a round-robin
|
||||
residue class is precisely DCP rank r's view.
|
||||
"""
|
||||
nr, _, heads, _ = q_f32.shape
|
||||
dev = q_f32.device
|
||||
o = torch.zeros(nr, qlen, heads, KV_LORA_RANK, dtype=torch.float32, device=dev)
|
||||
lse = torch.full((nr, qlen, heads), float("-inf"), dtype=torch.float32, device=dev)
|
||||
for r in range(nr):
|
||||
kv_sel = kv_f32[r].index_select(0, positions)
|
||||
for i in range(qlen):
|
||||
valid = positions <= (ctx + i)
|
||||
if not bool(valid.any()):
|
||||
continue
|
||||
k = kv_sel[valid]
|
||||
s = (q_f32[r, i] @ k.T) * scale
|
||||
m = s.max(dim=-1, keepdim=True).values
|
||||
p = torch.exp(s - m)
|
||||
denom = p.sum(dim=-1, keepdim=True)
|
||||
o[r, i] = (p / denom) @ k[:, :KV_LORA_RANK]
|
||||
lse[r, i] = m.squeeze(-1) + torch.log(denom.squeeze(-1))
|
||||
return o, lse
|
||||
|
||||
|
||||
def _merge_partials(os_, lses):
|
||||
"""Cross-shard softmax merge (natural-log LSE)."""
|
||||
stack_lse = torch.stack(lses)
|
||||
m = stack_lse.max(dim=0).values
|
||||
w = torch.exp(stack_lse - m)
|
||||
num = (w.unsqueeze(-1) * torch.stack(os_)).sum(dim=0)
|
||||
den = w.sum(dim=0).unsqueeze(-1)
|
||||
return num / den, m + torch.log(w.sum(dim=0))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# single-process rank simulation
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeGroup:
|
||||
def __init__(self, world_size: int, rank: int):
|
||||
self.world_size = world_size
|
||||
self.rank_in_group = rank
|
||||
self.rank = rank
|
||||
self.local_rank = rank
|
||||
|
||||
def all_gather(self, tensor, dim=0):
|
||||
raise RuntimeError(
|
||||
"collective called in a single-process test -- the path under test "
|
||||
"is supposed to be collective-free; this indicates a real change"
|
||||
)
|
||||
|
||||
|
||||
def _install_fake_groups(dcp_size: int, cp_rank: int, tp_size: int = 1) -> None:
|
||||
"""Point vLLM's parallel_state at rank-simulating stubs (no NCCL)."""
|
||||
import vllm.distributed.parallel_state as ps
|
||||
import vllm.model_executor.layers.attention.mla_attention as mla_attention
|
||||
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
|
||||
|
||||
dcp = _FakeGroup(dcp_size, cp_rank)
|
||||
tp = _FakeGroup(tp_size, cp_rank)
|
||||
ps.get_dcp_group = lambda: dcp
|
||||
ps.get_tp_group = lambda: tp
|
||||
# Both of these imported the symbol directly at module import time, so
|
||||
# patching parallel_state alone leaves them resolving the real group (and
|
||||
# the builder then reports dcp_world_size 1, silently skipping the route).
|
||||
backend.get_dcp_group = lambda: dcp
|
||||
mla_attention.get_dcp_group = lambda: dcp
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _restore_parallel_groups():
|
||||
"""Undo _install_fake_groups after every test.
|
||||
|
||||
It patches module globals in three modules, so without this the fake DCP
|
||||
group leaks into the rest of the pytest session and silently breaks
|
||||
unrelated tests that expect a real (or absent) group.
|
||||
"""
|
||||
import vllm.distributed.parallel_state as ps
|
||||
import vllm.model_executor.layers.attention.mla_attention as mla_attention
|
||||
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
|
||||
|
||||
saved = [
|
||||
(mod, name, getattr(mod, name))
|
||||
for mod, name in (
|
||||
(ps, "get_dcp_group"),
|
||||
(ps, "get_tp_group"),
|
||||
(backend, "get_dcp_group"),
|
||||
(mla_attention, "get_dcp_group"),
|
||||
)
|
||||
]
|
||||
yield
|
||||
for mod, name, value in saved:
|
||||
setattr(mod, name, value)
|
||||
|
||||
|
||||
class _PrefillBackendStub:
|
||||
"""The base builder does ``attention_layer.prefill_backend.clone()``.
|
||||
|
||||
prefill_backend is consulted only from the chunked-context prefill path,
|
||||
which a decode-only test never enters.
|
||||
"""
|
||||
|
||||
def clone(self):
|
||||
return self
|
||||
|
||||
|
||||
def _make_ctx_layer(vllm_config):
|
||||
"""Stand-in for the registered attention layer in static_forward_context.
|
||||
|
||||
``MLACommonMetadataBuilder.__init__`` reads the MLA dims, ``dcp_manager``
|
||||
and ``prefill_backend`` off this object. ``dcp_manager`` is the Direct-DCP
|
||||
symmetric-memory workspace, used ONLY by the chunked-prefill KV all-gather;
|
||||
it is constructed without running __init__ so no peer pointers are exchanged
|
||||
and no collective is issued. That is what keeps this test single-process.
|
||||
"""
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
|
||||
mgr = object.__new__(MLADCPManager)
|
||||
mgr.init_kv_gather = lambda *a, **k: None
|
||||
|
||||
return SimpleNamespace(
|
||||
non_causal_multi_token_decode=False,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=QK_NOPE_HEAD_DIM,
|
||||
dcp_manager=mgr,
|
||||
prefill_backend=_PrefillBackendStub(),
|
||||
)
|
||||
|
||||
|
||||
def _ensure_workspace_manager(device: torch.device) -> None:
|
||||
"""Init the global workspace manager, which GPUModelRunner does in production.
|
||||
|
||||
``AiterMLAMetadataBuilder.__init__`` reserves the fp8 prefill
|
||||
persistent-scheduling scratch through ``current_workspace_manager()``, so
|
||||
merely constructing a builder outside a serve trips its assert. Idempotent:
|
||||
one builder is constructed per DCP shard. Deliberately does NOT lock the
|
||||
workspace -- production locks only after warmup, and this keeps allocating
|
||||
across shards.
|
||||
"""
|
||||
from vllm.v1.worker.workspace import (
|
||||
init_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
if not is_workspace_manager_initialized():
|
||||
init_workspace_manager(device)
|
||||
|
||||
|
||||
class _Layer:
|
||||
def __init__(self, device):
|
||||
self._q_scale = torch.tensor(1.0, device=device)
|
||||
self._k_scale = torch.tensor(1.0, device=device)
|
||||
self._v_scale = torch.tensor(1.0, device=device)
|
||||
|
||||
|
||||
def _build_impl(num_heads: int, dcp: int, cp_rank: int):
|
||||
"""Real AiterMLAImpl.forward_mqa bound to a minimally-populated instance.
|
||||
|
||||
Constructing the full impl would drag in kv_b_proj and the rest of the
|
||||
weight-dependent MLA setup -- i.e. model loading, which this test exists to
|
||||
avoid. forward_mqa's fold path reads only the attributes set below.
|
||||
"""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM
|
||||
impl.num_heads = num_heads
|
||||
impl.dcp_world_size = dcp
|
||||
impl.dcp_rank = cp_rank
|
||||
impl.pcp_world_size = 1
|
||||
impl.kv_cache_dtype = "fp8"
|
||||
impl.scale = SCALE
|
||||
# `_decode_num_heads` is a read-only property deriving num_heads *
|
||||
# dcp_world_size, both set above; assert rather than assign.
|
||||
assert impl._decode_num_heads == num_heads * dcp
|
||||
return impl
|
||||
|
||||
|
||||
def _make_vllm_config(heads_per_rank: int):
|
||||
"""A DCP8 MLA config with no weights and no distributed init.
|
||||
|
||||
DeepSeek-R1's config supplies the MLA dims (kv_lora_rank 512, rope 64,
|
||||
nope 128) that the aiter asm kernel is built for; only the head count is
|
||||
overridden, to place the GATHERED count (heads_per_rank * DCP) on the
|
||||
kernel variant under test. ``get_num_attention_heads`` reads
|
||||
model_arch_config, a snapshot taken at config-creation time, so patching
|
||||
hf_config alone would be inert.
|
||||
"""
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name=MODEL,
|
||||
tensor_parallel_size=1,
|
||||
max_model_len=2048,
|
||||
max_num_seqs=NUM_REQS * 2,
|
||||
max_num_batched_tokens=4096,
|
||||
)
|
||||
vllm_config.model_config.model_arch_config.total_num_attention_heads = (
|
||||
heads_per_rank
|
||||
)
|
||||
# Without a speculative config the batch reorder threshold stays at 1 and a
|
||||
# qlen-QLEN row is classified as a PREFILL, so the decode path under test is
|
||||
# never reached. _init_reorder_batch_threshold reads exactly these two
|
||||
# fields, so a stand-in is enough and avoids pulling in a draft model.
|
||||
vllm_config.speculative_config = SimpleNamespace(
|
||||
num_speculative_tokens=QLEN - 1, parallel_drafting=False
|
||||
)
|
||||
vllm_config.cache_config.cache_dtype = "fp8"
|
||||
vllm_config.parallel_config.decode_context_parallel_size = DCP
|
||||
vllm_config.parallel_config.cp_kv_cache_interleave_size = 1
|
||||
return vllm_config
|
||||
|
||||
|
||||
def _run_shard(
|
||||
vllm_config,
|
||||
spec,
|
||||
layer_name,
|
||||
device,
|
||||
cp_rank,
|
||||
kern_rank,
|
||||
q_flat,
|
||||
kv_flat,
|
||||
nr,
|
||||
qlen,
|
||||
ctx,
|
||||
dcp,
|
||||
pos,
|
||||
):
|
||||
"""Build real metadata for one DCP rank and run the real asm kernel.
|
||||
|
||||
``kern_rank`` is separate from ``cp_rank`` so the positive control can hand
|
||||
rank r's pages to the kernel while telling it that it is rank r+1.
|
||||
"""
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
|
||||
|
||||
L = ctx + qlen
|
||||
local_len = int(pos.numel())
|
||||
|
||||
# One GLOBAL kv cache: page (req*L + p) holds request req position p. Shards
|
||||
# select pages by residue class, so every shard reads the same underlying
|
||||
# bytes and no re-quantisation noise is introduced between arms.
|
||||
block_table = (
|
||||
torch.arange(nr, device=device).view(nr, 1) * L + pos.view(1, -1)
|
||||
).to(torch.int32)
|
||||
|
||||
# seq_lens is the GLOBAL context; dcp_local_seq_lens is what this rank
|
||||
# actually holds. Passing the global length as the local one walks the
|
||||
# kernel off the end of the page table.
|
||||
seq_lens_cpu = torch.full((nr,), L, dtype=torch.int32)
|
||||
local_cpu = torch.full((nr,), local_len, dtype=torch.int32)
|
||||
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=torch.arange(
|
||||
0, nr * qlen + 1, qlen, dtype=torch.int32, device=device
|
||||
),
|
||||
query_start_loc_cpu=torch.arange(0, nr * qlen + 1, qlen, dtype=torch.int32),
|
||||
seq_lens=seq_lens_cpu.to(device),
|
||||
num_reqs=nr,
|
||||
num_actual_tokens=nr * qlen,
|
||||
max_query_len=qlen,
|
||||
max_seq_len=L,
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=torch.arange(nr * qlen, dtype=torch.int64, device=device),
|
||||
causal=True,
|
||||
dcp_local_seq_lens=local_cpu.to(device),
|
||||
dcp_local_seq_lens_cpu_upper_bound=local_cpu,
|
||||
)
|
||||
|
||||
_install_fake_groups(dcp, kern_rank)
|
||||
builder = AiterMLAMetadataBuilder(spec, [layer_name], vllm_config, device)
|
||||
md = builder.build(0, common)
|
||||
|
||||
impl = _build_impl(builder.num_heads, dcp, kern_rank)
|
||||
o, lse = impl.forward_mqa(q_flat, kv_flat, md, _Layer(device))
|
||||
torch.accelerator.synchronize()
|
||||
return md, o, lse, local_len
|
||||
|
||||
|
||||
def _numerics(heads_per_rank: int, sabotage: bool = False):
|
||||
"""Run every DCP shard and return (per-shard max rel err, merged rel err)."""
|
||||
import aiter
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
|
||||
device = torch.device("cuda", 0)
|
||||
torch.accelerator.set_device_index(device.index)
|
||||
torch.manual_seed(0)
|
||||
_ensure_workspace_manager(device)
|
||||
|
||||
_install_fake_groups(DCP, 0)
|
||||
vllm_config = _make_vllm_config(heads_per_rank)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
vllm_config.compilation_config.static_forward_context[layer_name] = (
|
||||
_make_ctx_layer(vllm_config)
|
||||
)
|
||||
# block_size 1: one page per token, so page ids are positions and the
|
||||
# residue-class page table is expressible directly.
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
non_causal_multi_token_decode=False,
|
||||
)
|
||||
|
||||
nr, ctx, qlen = NUM_REQS, CTX, QLEN
|
||||
L = ctx + qlen
|
||||
gathered_heads = heads_per_rank * DCP
|
||||
kv_dtype = aiter.dtypes.fp8
|
||||
|
||||
kv_flat = torch.randn(
|
||||
nr * L, 1, HEAD_SIZE, dtype=torch.float32, device=device
|
||||
).to(kv_dtype)
|
||||
q_flat = (
|
||||
torch.randn(
|
||||
nr * qlen,
|
||||
gathered_heads,
|
||||
HEAD_SIZE,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
.mul_(0.5)
|
||||
.to(kv_dtype)
|
||||
)
|
||||
# Dequantise for the reference so the comparison measures the kernel,
|
||||
# not the fp8 cast that both sides share.
|
||||
kv_f32 = kv_flat.float().view(nr, L, HEAD_SIZE)
|
||||
q_f32 = q_flat.float().view(nr, qlen, gathered_heads, HEAD_SIZE)
|
||||
|
||||
ref_o, _ = _torch_reference(
|
||||
q_f32, kv_f32, ctx, qlen, SCALE, torch.arange(L, device=device)
|
||||
)
|
||||
|
||||
shard_o, shard_lse = [], []
|
||||
max_shard_rel = 0.0
|
||||
for r in range(DCP):
|
||||
pos = torch.arange(r, L, DCP, device=device) # residue class r
|
||||
kern_rank = (r + 1) % DCP if sabotage else r
|
||||
md, o, lse, _ = _run_shard(
|
||||
vllm_config,
|
||||
spec,
|
||||
layer_name,
|
||||
device,
|
||||
r,
|
||||
kern_rank,
|
||||
q_flat,
|
||||
kv_flat,
|
||||
nr,
|
||||
qlen,
|
||||
ctx,
|
||||
DCP,
|
||||
pos,
|
||||
)
|
||||
# The route actually under test must have been taken; a silent
|
||||
# fall-out to Triton/segmented would otherwise pass quietly.
|
||||
assert md.decode.dcp_route is rocm_aiter_mla._DCPDecodeRoute.CPRR
|
||||
assert md.decode.asm_decode_num_heads, "cprr asm route was not taken"
|
||||
|
||||
o = o.float().view(nr, qlen, gathered_heads, KV_LORA_RANK)
|
||||
shard_lse.append(lse.float().view(nr, qlen, gathered_heads))
|
||||
shard_o.append(o)
|
||||
|
||||
po, _ = _torch_reference(q_f32, kv_f32, ctx, qlen, SCALE, pos)
|
||||
rel = (o - po).abs().max().item() / max(po.abs().max().item(), 1e-6)
|
||||
max_shard_rel = max(max_shard_rel, rel)
|
||||
|
||||
merged, _ = _merge_partials(shard_o, shard_lse)
|
||||
merge_rel = (merged - ref_o).abs().max().item() / max(
|
||||
ref_o.abs().max().item(), 1e-6
|
||||
)
|
||||
return max_shard_rel, merge_rel
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# tests
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@requires_cprr
|
||||
@pytest.mark.parametrize(
|
||||
"heads_per_rank",
|
||||
[
|
||||
16, # 128 gathered heads -- a NATIVE cprr kernel, no padding
|
||||
12, # 96 gathered heads -- K3's DSpark target, must pad up to 128
|
||||
],
|
||||
ids=["native-128", "padded-96-to-128"],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_cprr_shards_match_reference_and_merge_to_full_context(heads_per_rank):
|
||||
"""Every DCP shard must be right, and their LSE merge must reproduce
|
||||
full-context attention.
|
||||
|
||||
A shard that ignored global positions (applying a local causal tail
|
||||
instead) still produces plausible-looking output; it only diverges from the
|
||||
reference on the tokens whose causal window is truncated. Checking the merge
|
||||
alone would hide that, so both are asserted.
|
||||
"""
|
||||
shard_rel, merge_rel = _numerics(heads_per_rank)
|
||||
assert shard_rel < SHARD_RTOL, f"per-shard rel err {shard_rel:.3e}"
|
||||
assert merge_rel < MERGE_RTOL, f"merged rel err {merge_rel:.3e}"
|
||||
|
||||
|
||||
@requires_cprr
|
||||
@torch.inference_mode()
|
||||
def test_wrong_cp_rank_breaks_the_result():
|
||||
"""Positive control: without it, a passing run above proves nothing.
|
||||
|
||||
Feeding shard r's pages to the kernel while telling it that it is rank r+1
|
||||
shifts every reconstructed global position by one, so both the causal mask
|
||||
and the round-robin reconstruction are wrong. If the error does NOT blow up,
|
||||
the kernel is not using cp_rank and the test above has no power.
|
||||
"""
|
||||
shard_rel, merge_rel = _numerics(16, sabotage=True)
|
||||
assert shard_rel > SHARD_RTOL or merge_rel > MERGE_RTOL, (
|
||||
f"sabotaged run still passed (shard {shard_rel:.3e}, merge "
|
||||
f"{merge_rel:.3e}) -- cp_rank is not reaching the kernel"
|
||||
)
|
||||
|
||||
|
||||
@requires_cprr
|
||||
def test_cprr_kernel_is_selected_for_the_k3_shape():
|
||||
"""The K3 DSpark target shape must actually reach the cprr route.
|
||||
|
||||
Guards against a silent fall-out to segmented/Triton, which is what the
|
||||
stock configuration does and what this route exists to avoid.
|
||||
"""
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla as m
|
||||
|
||||
assert (
|
||||
m._asm_dcp_verify_configured(
|
||||
dcp_world_size=DCP, cp_interleave=1, multi_token_decode=True
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert QLEN >= m._MIN_CPRR_QLEN
|
||||
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
|
||||
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
|
||||
@@ -87,21 +87,31 @@ def _builder(
|
||||
kv_cache_dtype: str = "auto",
|
||||
dcp_world_size: int = 1,
|
||||
dcp_rank: int = 0,
|
||||
asm_dcp_verify: bool = False,
|
||||
decode_causal: bool = True,
|
||||
supports_segmented_dcp_verify: bool | None = None,
|
||||
):
|
||||
decode_num_heads = num_heads * dcp_world_size
|
||||
asm_heads = (
|
||||
rocm_aiter_mla._asm_dcp_verify_heads(decode_num_heads) if asm_dcp_verify else 0
|
||||
)
|
||||
if supports_segmented_dcp_verify is None:
|
||||
supports_segmented_dcp_verify = rocm_aiter_mla._segmented_dcp_verify_supported(
|
||||
dcp_world_size, 1
|
||||
)
|
||||
stub = SimpleNamespace(
|
||||
device=torch.device("cpu"),
|
||||
num_heads=num_heads,
|
||||
# DCP gathers every rank's head shard before decode, and the routing
|
||||
# predicates read the gathered count.
|
||||
_decode_num_heads=num_heads * dcp_world_size,
|
||||
_decode_num_heads=decode_num_heads,
|
||||
dcp_world_size=dcp_world_size,
|
||||
dcp_rank=dcp_rank,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
# Mirrors the production constructor: configuration decides whether the
|
||||
# segmented route is available, query length decides per batch.
|
||||
_supports_segmented_dcp_verify=rocm_aiter_mla._segmented_dcp_verify_supported(
|
||||
dcp_world_size, 1
|
||||
),
|
||||
# segmented route is available, query length decides per batch. CPRR
|
||||
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
|
||||
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
|
||||
# Derived once in the real constructor, so derive it once here too.
|
||||
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
|
||||
_dcp_verify_buffers=None,
|
||||
@@ -122,7 +132,11 @@ def _builder(
|
||||
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
|
||||
_use_persistent_metadata=False,
|
||||
kernel_block_size=kernel_block_size,
|
||||
_num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads),
|
||||
_num_attention_heads=(
|
||||
asm_heads
|
||||
if asm_dcp_verify
|
||||
else AiterMLAHelper.get_actual_mla_num_heads(num_heads)
|
||||
),
|
||||
_mla_work_meta_data=torch.empty(1, dtype=torch.int32),
|
||||
_mla_work_info_set=torch.empty(1, dtype=torch.int32),
|
||||
_mla_work_indptr=torch.empty(1, dtype=torch.int32),
|
||||
@@ -132,7 +146,19 @@ def _builder(
|
||||
_mla_q_dtype=torch.bfloat16,
|
||||
_mla_kv_dtype=torch.bfloat16,
|
||||
decode_attn_out_dtype=torch.bfloat16,
|
||||
_decode_causal=True,
|
||||
_decode_causal=decode_causal,
|
||||
# Default off so existing cases keep exercising segmented/persistent.
|
||||
# Pass asm_dcp_verify=True for the CPRR route and its qlen-floor fallback.
|
||||
_asm_dcp_verify=asm_dcp_verify,
|
||||
_asm_dcp_verify_heads=asm_heads,
|
||||
_g_kv_indptr_buf=(
|
||||
torch.zeros(max_decode_rows + 1, dtype=torch.int32)
|
||||
if asm_dcp_verify and dcp_world_size > 1
|
||||
else None
|
||||
),
|
||||
# Only read when the asm route is on; the real builder derives it from
|
||||
# the device CU count.
|
||||
_mla_max_split_per_batch=256,
|
||||
)
|
||||
# Bound to the stub rather than faked: the verify flatten's per-row view is
|
||||
# part of what _build_decode is being tested for.
|
||||
@@ -287,6 +313,209 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch):
|
||||
assert metadata.dcp_verify is not None
|
||||
|
||||
|
||||
def _cprr_decode_batch(builder, *, qlen: int, num_reqs: int = 1):
|
||||
seq_lens = torch.full((num_reqs,), 16, dtype=torch.int32)
|
||||
query_start_loc = torch.arange(0, (num_reqs + 1) * qlen, qlen, dtype=torch.int32)
|
||||
return AiterMLAMetadataBuilder._build_decode(
|
||||
builder,
|
||||
block_table_tensor=torch.zeros(num_reqs, 8, dtype=torch.int32),
|
||||
seq_lens_device=seq_lens,
|
||||
max_seq_len=16,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=num_reqs * qlen,
|
||||
max_query_len=qlen,
|
||||
dcp_tot_seq_lens_device=seq_lens * 8,
|
||||
)
|
||||
|
||||
|
||||
def test_asm_preferred_qlen2_falls_back_to_segmented(monkeypatch):
|
||||
"""K=4 can clamp to qlen 2; CPRR has no kernel, so use segmented MLA."""
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
|
||||
)
|
||||
|
||||
metadata = _cprr_decode_batch(
|
||||
_builder(
|
||||
mtp_decode_qlen=5,
|
||||
dcp_world_size=8,
|
||||
num_heads=12,
|
||||
asm_dcp_verify=True,
|
||||
kernel_block_size=2,
|
||||
),
|
||||
qlen=2,
|
||||
)
|
||||
|
||||
assert metadata.dcp_verify is not None
|
||||
assert metadata.g_kv_indptr is None
|
||||
assert not metadata.has_persistent_metadata
|
||||
get_mla_metadata_v1.assert_not_called()
|
||||
|
||||
|
||||
def test_asm_preferred_qlen_at_cprr_floor_stays_on_cprr(monkeypatch):
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
|
||||
metadata = _cprr_decode_batch(
|
||||
_builder(
|
||||
mtp_decode_qlen=5,
|
||||
dcp_world_size=8,
|
||||
num_heads=12,
|
||||
asm_dcp_verify=True,
|
||||
),
|
||||
qlen=rocm_aiter_mla._MIN_CPRR_QLEN,
|
||||
)
|
||||
|
||||
assert metadata.dcp_verify is None
|
||||
assert metadata.g_kv_indptr is not None
|
||||
assert metadata.has_persistent_metadata
|
||||
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is True
|
||||
|
||||
|
||||
def test_cprr_capable_single_token_decode_keeps_persistent_metadata(monkeypatch):
|
||||
"""The CPRR qlen floor must not suppress the ordinary qlen-1 schedule."""
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
|
||||
metadata = _cprr_decode_batch(
|
||||
_builder(
|
||||
mtp_decode_qlen=5,
|
||||
dcp_world_size=8,
|
||||
num_heads=12,
|
||||
asm_dcp_verify=True,
|
||||
),
|
||||
qlen=1,
|
||||
)
|
||||
|
||||
assert metadata.g_kv_indptr is None
|
||||
assert metadata.has_persistent_metadata
|
||||
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None
|
||||
|
||||
|
||||
def test_non_causal_dcp_block_uses_plain_mask0_metadata(monkeypatch):
|
||||
"""A non-causal block needs no CPRR global-position window or head pad."""
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
builder = _builder(
|
||||
mtp_decode_qlen=5,
|
||||
dcp_world_size=8,
|
||||
num_heads=12,
|
||||
asm_dcp_verify=True,
|
||||
)
|
||||
builder._decode_causal = False
|
||||
|
||||
metadata = _cprr_decode_batch(builder, qlen=4)
|
||||
|
||||
assert metadata.dcp_verify is None
|
||||
assert metadata.g_kv_indptr is None
|
||||
assert metadata.has_persistent_metadata
|
||||
assert get_mla_metadata_v1.call_args.args[3] == 96
|
||||
assert get_mla_metadata_v1.call_args.args[5] is False
|
||||
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None
|
||||
|
||||
|
||||
def test_asm_qlen2_without_segmented_fails_during_build(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
|
||||
_cprr_decode_batch(
|
||||
_builder(
|
||||
mtp_decode_qlen=5,
|
||||
dcp_world_size=8,
|
||||
num_heads=12,
|
||||
asm_dcp_verify=True,
|
||||
supports_segmented_dcp_verify=False,
|
||||
),
|
||||
qlen=2,
|
||||
)
|
||||
|
||||
|
||||
def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
|
||||
"""Process-level CPRR capability must not pad or route a non-causal batch."""
|
||||
captured = {}
|
||||
|
||||
def fake_aiter_decode(q, kv_buffer, out, *args, **kwargs):
|
||||
captured["q_heads"] = q.shape[1]
|
||||
captured["kwargs"] = kwargs
|
||||
return None, torch.zeros(q.shape[0], q.shape[1])
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_get_aiter_mla_decode", lambda: fake_aiter_decode
|
||||
)
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.num_heads = 12
|
||||
impl.dcp_world_size = 8
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.kv_lora_rank = 512
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.scale = 576**-0.5
|
||||
decode = SimpleNamespace(
|
||||
max_qo_len=4,
|
||||
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
|
||||
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
|
||||
paged_kv_indices=torch.zeros(1, dtype=torch.int32),
|
||||
paged_kv_last_page_len=torch.ones(1, dtype=torch.int32),
|
||||
use_gluon_decode=False,
|
||||
use_gluon_verify=False,
|
||||
dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN,
|
||||
dcp_verify=None,
|
||||
g_kv_indptr=None,
|
||||
has_persistent_metadata=False,
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
asm_decode_num_heads=0,
|
||||
mla_num_kv_splits=0,
|
||||
cp_world_size=8,
|
||||
cp_rank=0,
|
||||
min_kv_seq_len=1,
|
||||
)
|
||||
attn_metadata = SimpleNamespace(decode=decode, causal=False, work_meta_data=None)
|
||||
layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0))
|
||||
q = torch.zeros(4, 96, 576, dtype=torch.bfloat16)
|
||||
|
||||
output, lse = impl.forward_mqa(q, torch.zeros(1, 1, 576), attn_metadata, layer)
|
||||
|
||||
assert captured["q_heads"] == 96
|
||||
assert captured["kwargs"]["causal"] is False
|
||||
assert "g_kv_indptr" not in captured["kwargs"]
|
||||
assert "num_kv_splits" not in captured["kwargs"]
|
||||
assert output.shape[1] == 96
|
||||
assert lse is not None and lse.shape == (4, 96)
|
||||
|
||||
|
||||
def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
|
||||
"""Single-token DCP decode must come back with an LSE the merge can use.
|
||||
|
||||
@@ -325,9 +554,14 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
|
||||
paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32),
|
||||
use_gluon_decode=False,
|
||||
use_gluon_verify=False,
|
||||
dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN,
|
||||
dcp_verify=None,
|
||||
has_persistent_metadata=False,
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
# 0 == the round-robin (cprr) asm route is off, this rank's decode takes
|
||||
# the existing path. Mirrors the AiterMLADecodeMetadata default.
|
||||
asm_decode_num_heads=0,
|
||||
g_kv_indptr=None,
|
||||
)
|
||||
attn_metadata = SimpleNamespace(decode=decode, causal=True, work_meta_data=None)
|
||||
layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0))
|
||||
@@ -453,6 +687,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
),
|
||||
use_gluon_decode=False,
|
||||
use_gluon_verify=False,
|
||||
dcp_route=rocm_aiter_mla._DCPDecodeRoute.SEGMENTED,
|
||||
dcp_verify=view,
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user