[ROCm][MLA] Keep DCP prefill context in FP8 through AllGather (#59070)
Signed-off-by: whx-sjtu <xiaowang990929@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Code <noreply@anthropic.com> Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Code
Fangzhou Ai
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Simon Danielsson
parent
98f7211162
commit
c6566bb9f6
@@ -3539,8 +3539,11 @@ steps:
|
||||
- examples/rl/rlhf_async_new_apis.py
|
||||
- tests/distributed/test_weight_transfer.py
|
||||
- tests/distributed/test_packed_tensor.py
|
||||
- tests/v1/attention/test_rocm_mla_dcp_context.py
|
||||
- vllm/v1/attention/ops/rocm_aiter_mla_prefill.py
|
||||
commands:
|
||||
- pytest -v -s tests/distributed/test_context_parallel.py
|
||||
- pytest -v -s tests/v1/attention/test_rocm_mla_dcp_context.py
|
||||
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
|
||||
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
|
||||
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
|
||||
|
||||
@@ -528,8 +528,11 @@ steps:
|
||||
- examples/rl/rlhf_async_new_apis.py
|
||||
- tests/distributed/test_weight_transfer.py
|
||||
- tests/distributed/test_packed_tensor.py
|
||||
- tests/v1/attention/test_rocm_mla_dcp_context.py
|
||||
- vllm/v1/attention/ops/rocm_aiter_mla_prefill.py
|
||||
commands:
|
||||
- pytest -v -s tests/distributed/test_context_parallel.py
|
||||
- pytest -v -s tests/v1/attention/test_rocm_mla_dcp_context.py
|
||||
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
|
||||
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
|
||||
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
|
||||
|
||||
@@ -33,6 +33,26 @@ def _require_aiter():
|
||||
pytest.skip("aiter is required on supported ROCm hardware for this test")
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_gather_kv_b_proj_op_schema() -> None:
|
||||
"""Validate fake tensors and mutation of both expanded K/V outputs."""
|
||||
_require_aiter()
|
||||
from vllm._aiter_ops import rocm_aiter_ops # noqa: F401
|
||||
|
||||
rows, num_heads = 32, 12
|
||||
cache = torch.randn(rows, 1, 576, device="cuda").to(current_platform.fp8_dtype())
|
||||
scale = torch.tensor([0.07], device="cuda")
|
||||
cu_seq_lens = torch.tensor([0, 13, rows], dtype=torch.int32, device="cuda")
|
||||
indices = torch.arange(rows, dtype=torch.int32, device="cuda").flip(0)
|
||||
weight = torch.randn(num_heads * 256, 512, dtype=torch.bfloat16, device="cuda")
|
||||
k = torch.empty(rows, num_heads, 192, dtype=torch.bfloat16, device="cuda")
|
||||
v = torch.empty(rows, num_heads, 128, dtype=torch.bfloat16, device="cuda")
|
||||
opcheck(
|
||||
torch.ops.vllm.rocm_aiter_gather_kv_b_proj,
|
||||
(cache, scale, cu_seq_lens, indices, cu_seq_lens, weight, k, v),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_fwd_op_schema() -> None:
|
||||
"""Opcheck validates registration, schema, fake-tensor, and ``o`` aliasing.
|
||||
|
||||
@@ -142,6 +142,13 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True):
|
||||
|
||||
def init_common_builder(self, *args, **kwargs):
|
||||
self.num_heads = gathered_heads // DCP
|
||||
self.q_data_type = torch.bfloat16
|
||||
self.mla_dims = SimpleNamespace(
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
)
|
||||
self.dcp_world_size = DCP
|
||||
self.reorder_batch_threshold = 5
|
||||
|
||||
|
||||
@@ -740,26 +740,34 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
torch.testing.assert_close(output, reference, rtol=3e-2, atol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [8, 16, 24, 32, 64, 128])
|
||||
@pytest.mark.parametrize(
|
||||
"spec_method, parallel_drafting",
|
||||
"num_heads,spec_method,parallel_drafting,group_cache_dtype,needs_row_map",
|
||||
[
|
||||
("deepseek_mtp", False),
|
||||
# A drafter that is not one of the historically recognized MTP methods,
|
||||
# and a parallel one, so the threshold is 1 + 2 * num_spec rather than
|
||||
# 1 + num_spec. Sizing the metadata off a method name instead leaves
|
||||
# these at qlen=1 while the router still admits the full range.
|
||||
("custom", True),
|
||||
("eagle", False),
|
||||
*[
|
||||
(num_heads, method, parallel, None, True)
|
||||
for num_heads in [8, 16, 24, 32, 64, 128]
|
||||
for method, parallel in [
|
||||
("deepseek_mtp", False),
|
||||
# Parallel drafting needs qlen=1+2*num_spec, regardless of name.
|
||||
("custom", True),
|
||||
("eagle", False),
|
||||
]
|
||||
],
|
||||
pytest.param(16, "deepseek_mtp", False, "auto", False, id="bf16-group"),
|
||||
pytest.param(16, "deepseek_mtp", False, "fp8_e5m2", False, id="e5m2-group"),
|
||||
],
|
||||
)
|
||||
def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
monkeypatch, num_heads, spec_method, parallel_drafting
|
||||
def test_builder_init_sizes_decode_metadata_and_filters_dcp_maps(
|
||||
monkeypatch,
|
||||
num_heads,
|
||||
spec_method,
|
||||
parallel_drafting,
|
||||
group_cache_dtype,
|
||||
needs_row_map,
|
||||
):
|
||||
"""Aiter init sizes the metadata for every query length decode can be handed.
|
||||
"""Size decode metadata for padded heads/qlen; skip unsupported DCP maps.
|
||||
|
||||
Sweeping num_heads asserts metadata is sized for the padded decode shape,
|
||||
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
|
||||
The group's raw cache format must override the global FP8 configuration.
|
||||
"""
|
||||
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
|
||||
info_calls = []
|
||||
@@ -789,6 +797,13 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
|
||||
def init_common_builder(self, *args, **kwargs):
|
||||
self.num_heads = num_heads
|
||||
self.q_data_type = config.model_config.dtype
|
||||
self.mla_dims = SimpleNamespace(
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
)
|
||||
self.dcp_world_size = 1
|
||||
self.cp_kv_cache_interleave_size = 1
|
||||
# Mirror what _init_reorder_batch_threshold would have left behind: the
|
||||
@@ -813,6 +828,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
init_common_builder,
|
||||
)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "is_aiter_found_and_supported", lambda: True)
|
||||
|
||||
config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
@@ -840,7 +856,9 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
),
|
||||
)
|
||||
builder = AiterMLAMetadataBuilder(
|
||||
kv_cache_spec=SimpleNamespace(block_size=1, dtype=torch.bfloat16),
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
block_size=1, dtype=torch.bfloat16, cache_dtype_str=group_cache_dtype
|
||||
),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=config,
|
||||
device=torch.device("cpu"),
|
||||
@@ -860,6 +878,35 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
assert builder._mla_q_dtype == dtypes.fp8
|
||||
assert builder._mla_kv_dtype == dtypes.fp8
|
||||
|
||||
builder.dcp_world_size = 2
|
||||
chunk = object()
|
||||
rows = torch.tensor([0], dtype=torch.int64)
|
||||
build_rows = mock.Mock(return_value=rows)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "context_row_indices", build_rows)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_supports_fused_dcp_prefill_config",
|
||||
mock.Mock(side_effect=AssertionError("Support must be cached at init")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla.MLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args: SimpleNamespace(
|
||||
prefill=SimpleNamespace(
|
||||
chunked_context=SimpleNamespace(chunks=[chunk]),
|
||||
block_table=torch.empty(0),
|
||||
),
|
||||
decode=None,
|
||||
dcp_context_row_indices=None,
|
||||
),
|
||||
)
|
||||
for _ in range(2):
|
||||
metadata = builder.build(0, SimpleNamespace(causal=True))
|
||||
assert metadata.dcp_context_row_indices == ([rows] if needs_row_map else None)
|
||||
assert build_rows.call_args_list == (
|
||||
[mock.call(chunk, torch.device("cpu"))] * 2 if needs_row_map else []
|
||||
)
|
||||
|
||||
|
||||
def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
|
||||
@@ -0,0 +1,445 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import multi_gpu_only
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_prefill import context_row_indices
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"padded,lengths,starts,total,expected",
|
||||
[
|
||||
([3, 2], [[7, 6], [2, 1]], [5, 0], 6, [0, 1, 5, 3, 4, 8]),
|
||||
([3], [[2, 9]], [4], 3, [3, 4, 5]),
|
||||
([3], [[1, 1]], [0], 3, None),
|
||||
],
|
||||
ids=["padding_and_chunk_starts", "empty_rank", "inconsistent_total"],
|
||||
)
|
||||
def test_dcp_context_indices(padded, lengths, starts, total, expected):
|
||||
chunk = SimpleNamespace(
|
||||
padded_local_seq_lens=padded,
|
||||
local_context_lens_allranks=lengths,
|
||||
local_starts=starts,
|
||||
num_local_context_tokens=sum(padded),
|
||||
num_context_tokens=total,
|
||||
)
|
||||
if expected is None:
|
||||
with pytest.raises(AssertionError):
|
||||
context_row_indices(chunk, torch.device("cpu"))
|
||||
else:
|
||||
rows = context_row_indices(chunk, torch.device("cpu"))
|
||||
assert rows.tolist() == expected
|
||||
assert rows.dtype == torch.int32
|
||||
|
||||
|
||||
def test_compressed_gather_preserves_bytes_and_workspace_partition(monkeypatch):
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_prefill as prefill
|
||||
|
||||
workspace = torch.full((12, 4), -1, dtype=torch.bfloat16)
|
||||
cache = torch.arange(8, dtype=torch.uint8).reshape(2, 4)
|
||||
chunk = SimpleNamespace(
|
||||
num_local_context_tokens=2,
|
||||
local_context_lens_allranks=[[2, 2]],
|
||||
padded_local_cu_seq_lens=torch.tensor([0, 2]),
|
||||
num_requests=1,
|
||||
starts=torch.tensor([0]),
|
||||
)
|
||||
|
||||
def local_gather(src, dst, *args):
|
||||
assert src.dtype == dst.dtype == torch.uint8
|
||||
dst.copy_(src)
|
||||
|
||||
def allgather(dst, src):
|
||||
assert dst.dtype == src.dtype == torch.uint8
|
||||
assert dst.shape == (4, 4)
|
||||
assert src.shape == (2, 4)
|
||||
dst[:2].copy_(src)
|
||||
dst[2:].copy_(src + 16)
|
||||
|
||||
monkeypatch.setattr(prefill.ops, "cp_gather_cache", local_gather)
|
||||
gathered = prefill.gather_compressed_context(
|
||||
cache, workspace, torch.empty(1, 1), chunk, allgather, torch.float8_e4m3fn
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
gathered.view(torch.uint8), torch.cat((cache, cache + 16))
|
||||
)
|
||||
# Retain the original logical capacity even though the byte view is larger.
|
||||
assert gathered.data_ptr() - workspace.data_ptr() == 4 * 4
|
||||
torch.testing.assert_close(workspace.view(torch.uint8).reshape(-1, 4)[:2], cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unsupported",
|
||||
[
|
||||
"packed_cache",
|
||||
"e5m2",
|
||||
"bf16_cache",
|
||||
"quantized_weight",
|
||||
"bias",
|
||||
"no_indices",
|
||||
"query_dtype",
|
||||
"prefill_dtype",
|
||||
"weight_dtype",
|
||||
"weight_layout",
|
||||
"cache_layout",
|
||||
"dimensions",
|
||||
"no_aiter",
|
||||
],
|
||||
)
|
||||
def test_dcp_prefill_unsupported_formats_use_original_path(monkeypatch, unsupported):
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm MLA backend")
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
AiterMLAImpl,
|
||||
MLACommonImpl,
|
||||
)
|
||||
|
||||
if unsupported == "no_aiter":
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.rocm_aiter_mla."
|
||||
"is_aiter_found_and_supported",
|
||||
lambda: False,
|
||||
)
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.kv_cache_dtype = {
|
||||
"packed_cache": "fp8_ds_mla",
|
||||
"e5m2": "fp8_e5m2",
|
||||
"bf16_cache": "auto",
|
||||
}.get(unsupported, "fp8")
|
||||
impl.kv_lora_rank = 512
|
||||
if unsupported == "dimensions":
|
||||
impl.kv_lora_rank = 256
|
||||
impl.qk_nope_head_dim = impl.v_head_dim = 128
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.num_heads = 1
|
||||
impl.kv_b_proj = SimpleNamespace(
|
||||
weight=torch.empty(256, 512, dtype=torch.bfloat16),
|
||||
weight_scale=torch.ones(1) if unsupported == "quantized_weight" else None,
|
||||
bias=torch.ones(256) if unsupported == "bias" else None,
|
||||
)
|
||||
if unsupported == "weight_dtype":
|
||||
impl.kv_b_proj.weight = impl.kv_b_proj.weight.half()
|
||||
elif unsupported == "weight_layout":
|
||||
impl.kv_b_proj.weight = torch.empty(512, 256, dtype=torch.bfloat16).T
|
||||
metadata = SimpleNamespace(
|
||||
prefill=SimpleNamespace(
|
||||
q_data_type=torch.float16
|
||||
if unsupported == "prefill_dtype"
|
||||
else torch.bfloat16
|
||||
),
|
||||
dcp_context_row_indices=None if unsupported == "no_indices" else [],
|
||||
)
|
||||
expected = (torch.ones(1), torch.zeros(1))
|
||||
|
||||
def fused_mla_kv_concat(kv_nope, k_pe, use_fp8_prefill):
|
||||
raise AssertionError("The fallback should receive the callback")
|
||||
|
||||
def original_path(*args):
|
||||
assert args[-1] is fused_mla_kv_concat
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(
|
||||
MLACommonImpl,
|
||||
"_context_parallel_compute_prefill_context",
|
||||
original_path,
|
||||
)
|
||||
q = torch.empty(1, 1, 192, dtype=torch.bfloat16)
|
||||
cache = torch.empty(1, 1536, 576, dtype=torch.uint8)
|
||||
if unsupported == "query_dtype":
|
||||
q = q.half()
|
||||
elif unsupported == "cache_layout":
|
||||
cache = torch.empty(1, 1536, 1152, dtype=torch.uint8)[..., ::2]
|
||||
assert (
|
||||
impl._context_parallel_compute_prefill_context(
|
||||
q,
|
||||
cache,
|
||||
metadata,
|
||||
torch.ones(1),
|
||||
8,
|
||||
fused_mla_kv_concat_fn=fused_mla_kv_concat,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm MLA backend")
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
@pytest.mark.parametrize("interleave", [1, 64])
|
||||
@pytest.mark.parametrize(
|
||||
"context_lens,workspace_size",
|
||||
[([145, 37], 1024), ([545, 0, 3, 0], 256)],
|
||||
ids=["single_chunk", "continuations_and_empty_context"],
|
||||
)
|
||||
def test_dcp_prefill_matches_full_attention(
|
||||
interleave, context_lens, workspace_size, tmp_path
|
||||
):
|
||||
"""Real DCP gather, fused expansion and suffix/chunk merges match dense SDPA."""
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip("AITER on CDNA 3 or newer is required")
|
||||
mp.spawn(
|
||||
_dcp_prefill_worker,
|
||||
args=(
|
||||
f"file://{tmp_path / 'dist_init'}",
|
||||
interleave,
|
||||
context_lens,
|
||||
workspace_size,
|
||||
),
|
||||
nprocs=2,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def _dcp_prefill_worker(rank, init_method, interleave, context_lens, workspace_size):
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.v1.attention.test_mla_backends import MockMLAAttentionLayer
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import (
|
||||
AiterFlashAttnPrefillBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
AiterMLAImpl,
|
||||
AiterMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
torch.accelerator.set_device_index(rank)
|
||||
torch.set_num_threads(1)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
init_workspace_manager(device)
|
||||
world_size, num_heads, block_size = 2, 12, 64
|
||||
config = create_vllm_config(
|
||||
model_name="deepseek-ai/DeepSeek-V2-Lite",
|
||||
dtype=torch.bfloat16,
|
||||
tensor_parallel_size=world_size,
|
||||
max_model_len=2048,
|
||||
block_size=block_size,
|
||||
max_num_seqs=8,
|
||||
max_num_batched_tokens=2048,
|
||||
)
|
||||
config.cache_config.cache_dtype = "fp8"
|
||||
config.parallel_config.decode_context_parallel_size = world_size
|
||||
config.parallel_config.cp_kv_cache_interleave_size = interleave
|
||||
config.attention_config.use_prefill_query_quantization = False
|
||||
config.kernel_config.enable_jit_warmup = False
|
||||
with set_current_vllm_config(config):
|
||||
init_distributed_environment(
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
local_rank=rank,
|
||||
distributed_init_method=init_method,
|
||||
)
|
||||
initialize_model_parallel(
|
||||
tensor_model_parallel_size=world_size,
|
||||
decode_context_model_parallel_size=world_size,
|
||||
)
|
||||
try:
|
||||
projection = ColumnParallelLinear(
|
||||
512,
|
||||
num_heads * world_size * 256,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
).to(device)
|
||||
torch.manual_seed(101 + rank)
|
||||
projection.weight.copy_(torch.randn_like(projection.weight) * 0.02)
|
||||
dims = dict(
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
)
|
||||
impl = AiterMLAImpl(
|
||||
num_heads=num_heads,
|
||||
head_size=576,
|
||||
scale=192**-0.5,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="fp8",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
q_lora_rank=None,
|
||||
qk_head_dim=192,
|
||||
kv_b_proj=projection,
|
||||
**dims,
|
||||
)
|
||||
layer = MockMLAAttentionLayer(
|
||||
impl=impl,
|
||||
num_heads=num_heads,
|
||||
device=device,
|
||||
kv_b_proj=projection,
|
||||
q_scale=1.0,
|
||||
k_scale=0.07,
|
||||
**dims,
|
||||
)
|
||||
layer.prefill_backend = AiterFlashAttnPrefillBackend(
|
||||
num_heads=num_heads,
|
||||
scale=impl.scale,
|
||||
vllm_config=config,
|
||||
**dims,
|
||||
)
|
||||
layer.dcp_manager = MLADCPManager(
|
||||
config,
|
||||
device,
|
||||
num_heads,
|
||||
576,
|
||||
512,
|
||||
torch.bfloat16,
|
||||
torch.bfloat16,
|
||||
None,
|
||||
True,
|
||||
False,
|
||||
)
|
||||
config.compilation_config.static_forward_context["mla"] = layer
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=current_platform.fp8_dtype(),
|
||||
cache_dtype_str="fp8",
|
||||
)
|
||||
builder = AiterMLAMetadataBuilder(spec, ["mla"], config, device)
|
||||
builder.chunked_prefill_workspace_size = workspace_size
|
||||
builder.chunked_prefill_workspace = torch.empty(
|
||||
(workspace_size + workspace_size // world_size, 576),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
layer.dcp_manager.init_kv_gather(
|
||||
builder.chunked_prefill_workspace, workspace_size
|
||||
)
|
||||
query_lens = [i + 2 for i in range(len(context_lens))]
|
||||
batch = BatchSpec(
|
||||
seq_lens=[c + q for c, q in zip(context_lens, query_lens)],
|
||||
query_lens=query_lens,
|
||||
)
|
||||
common = create_common_attn_metadata(
|
||||
batch, block_size, device, arange_block_indices=True
|
||||
)
|
||||
metadata = builder.build(0, common)
|
||||
assert metadata.prefill is not None
|
||||
chunked = metadata.prefill.chunked_context
|
||||
assert chunked is not None and metadata.dcp_context_row_indices is not None
|
||||
if 0 in context_lens:
|
||||
assert chunked.empty_token_slices
|
||||
assert any(c.is_continuation for c in chunked.chunks)
|
||||
else:
|
||||
assert len(chunked.chunks) == 1
|
||||
blocks_per_req = common.block_table_tensor.shape[1]
|
||||
cache = torch.zeros(
|
||||
(len(context_lens) * blocks_per_req, block_size, 576),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
).to(current_platform.fp8_dtype())
|
||||
queries, suffixes, references = [], [], []
|
||||
# All ranks reconstruct the same global cache before sharding it.
|
||||
torch.manual_seed(42)
|
||||
for request, (context_len, query_len) in enumerate(
|
||||
zip(context_lens, query_lens)
|
||||
):
|
||||
latent = torch.randn(
|
||||
context_len, 576, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
quantized = (latent.float() / layer._k_scale).to(cache.dtype)
|
||||
positions = torch.arange(context_len, device=device)
|
||||
shard = quantized.view(torch.uint8)[
|
||||
(positions // interleave) % world_size == rank
|
||||
]
|
||||
cache.view(len(context_lens), -1, 576).view(torch.uint8)[
|
||||
request, : len(shard)
|
||||
].copy_(shard)
|
||||
suffix = torch.randn(
|
||||
query_len, 576, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
q = (
|
||||
torch.randn(
|
||||
query_len, num_heads, 192, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
* 0.5
|
||||
)
|
||||
full_latent = torch.cat(
|
||||
(
|
||||
(quantized.float() * layer._k_scale).to(torch.bfloat16),
|
||||
suffix,
|
||||
)
|
||||
)
|
||||
kv = F.linear(full_latent[:, :512], projection.weight).view(
|
||||
-1, num_heads, 256
|
||||
)
|
||||
k_nope, v = kv.split(128, dim=-1)
|
||||
k = torch.cat(
|
||||
(k_nope, full_latent[:, None, 512:].expand(-1, num_heads, -1)),
|
||||
dim=-1,
|
||||
)
|
||||
mask = torch.arange(context_len + query_len, device=device)[
|
||||
None, :
|
||||
] <= (context_len + torch.arange(query_len, device=device)[:, None])
|
||||
references.append(
|
||||
F.scaled_dot_product_attention(
|
||||
q.transpose(0, 1).float(),
|
||||
k.transpose(0, 1).float(),
|
||||
v.transpose(0, 1).float(),
|
||||
attn_mask=mask,
|
||||
scale=impl.scale,
|
||||
).transpose(0, 1)
|
||||
)
|
||||
queries.append(q)
|
||||
suffixes.append(suffix)
|
||||
q, suffix = torch.cat(queries), torch.cat(suffixes)
|
||||
output = torch.empty(
|
||||
q.shape[0], num_heads * 128, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
# Ensure this test cannot silently succeed through the fallback.
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
|
||||
def unexpected_fallback(*args, **kwargs):
|
||||
raise AssertionError("Expected the compressed DCP path")
|
||||
|
||||
patch.setattr(
|
||||
"vllm.model_executor.layers.attention.mla_attention."
|
||||
"MLACommonImpl._context_parallel_compute_prefill_context",
|
||||
unexpected_fallback,
|
||||
)
|
||||
impl.forward_mha(
|
||||
q,
|
||||
suffix[:, :512],
|
||||
suffix[:, None, 512:],
|
||||
cache,
|
||||
metadata,
|
||||
layer._k_scale,
|
||||
output,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output.view(-1, num_heads, 128).float(),
|
||||
torch.cat(references),
|
||||
rtol=0.03,
|
||||
atol=0.003,
|
||||
)
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
@@ -643,6 +643,36 @@ def _check_aiter_mla_fp8_support() -> bool:
|
||||
return _AITER_MLA_SUPPORTS_FP8
|
||||
|
||||
|
||||
def _rocm_aiter_gather_kv_b_proj_impl(
|
||||
kv_cache: torch.Tensor,
|
||||
kv_scale: torch.Tensor,
|
||||
kv_indptr: torch.Tensor,
|
||||
kv_indices: torch.Tensor,
|
||||
cu_seq_lens: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
) -> None:
|
||||
from aiter.ops.triton.gather_kv_b_proj import gather_kv_b_proj
|
||||
|
||||
gather_kv_b_proj(
|
||||
kv_cache, kv_scale, kv_indptr, kv_indices, cu_seq_lens, weight, None, k, v
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_gather_kv_b_proj_fake(
|
||||
kv_cache: torch.Tensor,
|
||||
kv_scale: torch.Tensor,
|
||||
kv_indptr: torch.Tensor,
|
||||
kv_indices: torch.Tensor,
|
||||
cu_seq_lens: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _rocm_aiter_mla_decode_fwd_impl(
|
||||
q: torch.Tensor,
|
||||
kv_buffer: torch.Tensor,
|
||||
@@ -2851,6 +2881,13 @@ class rocm_aiter_ops:
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_gather_kv_b_proj",
|
||||
op_func=_rocm_aiter_gather_kv_b_proj_impl,
|
||||
mutates_args=["k", "v"],
|
||||
fake_impl=_rocm_aiter_gather_kv_b_proj_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_mla_decode_fwd",
|
||||
op_func=_rocm_aiter_mla_decode_fwd_impl,
|
||||
@@ -3465,6 +3502,21 @@ class rocm_aiter_ops:
|
||||
extra_indices=extra_kv_indices,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def gather_kv_b_proj(
|
||||
kv_cache: torch.Tensor,
|
||||
kv_scale: torch.Tensor,
|
||||
kv_indptr: torch.Tensor,
|
||||
kv_indices: torch.Tensor,
|
||||
cu_seq_lens: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
) -> None:
|
||||
torch.ops.vllm.rocm_aiter_gather_kv_b_proj(
|
||||
kv_cache, kv_scale, kv_indptr, kv_indices, cu_seq_lens, weight, k, v
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def mla_decode_fwd(
|
||||
q: torch.Tensor,
|
||||
|
||||
@@ -2,15 +2,16 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, ClassVar, Final
|
||||
from typing import TYPE_CHECKING, ClassVar, Final, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_dcp_group
|
||||
@@ -22,6 +23,8 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonMetadata,
|
||||
MLACommonMetadataBuilder,
|
||||
QueryLenSupport,
|
||||
accumulate_mla_context_chunk,
|
||||
init_mla_context_partial,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -38,6 +41,11 @@ from vllm.v1.attention.backends.utils import (
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_merge import (
|
||||
merge_mla_segments_triton,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_prefill import (
|
||||
context_row_indices,
|
||||
expand_context,
|
||||
gather_compressed_context,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -46,6 +54,26 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _supports_fused_dcp_prefill_config(
|
||||
kv_cache_dtype: str | None,
|
||||
query_dtype: torch.dtype,
|
||||
kv_lora_rank: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
) -> bool:
|
||||
"""Whether the configuration supports fused DCP context expansion."""
|
||||
return (
|
||||
kv_cache_dtype in ("fp8", "fp8_e4m3")
|
||||
and query_dtype == torch.bfloat16
|
||||
and kv_lora_rank == 512
|
||||
and qk_nope_head_dim == 128
|
||||
and qk_rope_head_dim == 64
|
||||
and v_head_dim == 128
|
||||
and is_aiter_found_and_supported()
|
||||
)
|
||||
|
||||
|
||||
def _segmented_mla_page_size(block_size: int) -> int:
|
||||
"""Largest supported power-of-two subpage dividing a physical KV block.
|
||||
|
||||
@@ -507,6 +535,7 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
|
||||
|
||||
@dataclass
|
||||
class AiterMLAMetadata(MLACommonMetadata[AiterMLADecodeMetadata]):
|
||||
dcp_context_row_indices: list[torch.Tensor] | None = None
|
||||
work_meta_data: torch.Tensor | None = None
|
||||
work_indptr: torch.Tensor | None = None
|
||||
work_info_set: torch.Tensor | None = None
|
||||
@@ -664,6 +693,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
device
|
||||
).multi_processor_count
|
||||
|
||||
# Use the group's original format: decode normalizes E5M2 to "fp8".
|
||||
kv_cache_dtype = getattr(kv_cache_spec, "cache_dtype_str", None)
|
||||
if kv_cache_dtype is None:
|
||||
kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
self._supports_fused_dcp_prefill = _supports_fused_dcp_prefill_config(
|
||||
kv_cache_dtype,
|
||||
self.q_data_type,
|
||||
self.mla_dims.kv_lora_rank,
|
||||
self.mla_dims.qk_nope_head_dim,
|
||||
self.mla_dims.qk_rope_head_dim,
|
||||
self.mla_dims.v_head_dim,
|
||||
)
|
||||
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.decode_attn_out_dtype = vllm_config.model_config.dtype
|
||||
|
||||
@@ -1577,6 +1619,17 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
attn_metadata = super().build(
|
||||
common_prefix_len, common_attn_metadata, fast_build
|
||||
)
|
||||
prefill = attn_metadata.prefill
|
||||
if (
|
||||
prefill is not None
|
||||
and prefill.chunked_context is not None
|
||||
and self.dcp_world_size > 1
|
||||
and self._supports_fused_dcp_prefill
|
||||
):
|
||||
attn_metadata.dcp_context_row_indices = [
|
||||
context_row_indices(chunk, prefill.block_table.device)
|
||||
for chunk in prefill.chunked_context.chunks
|
||||
]
|
||||
if (
|
||||
attn_metadata.decode is not None
|
||||
and attn_metadata.decode.has_persistent_metadata
|
||||
@@ -1975,6 +2028,101 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
self._mla_prefill_ps_asm_fwd = mla_prefill_ps_asm_fwd
|
||||
self._mla_reduce_v1 = mla_reduce_v1
|
||||
|
||||
def _context_parallel_compute_prefill_context(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: MLACommonMetadata,
|
||||
k_scale: torch.Tensor,
|
||||
dcp_world_size: int,
|
||||
fused_mla_kv_concat_fn: Callable[
|
||||
[torch.Tensor, torch.Tensor, bool], tuple[torch.Tensor, torch.Tensor]
|
||||
]
|
||||
| None = None,
|
||||
):
|
||||
attn_metadata = cast(AiterMLAMetadata, attn_metadata)
|
||||
assert attn_metadata.prefill is not None
|
||||
prefill = attn_metadata.prefill
|
||||
weight = getattr(self.kv_b_proj, "weight", None)
|
||||
eligible = (
|
||||
_supports_fused_dcp_prefill_config(
|
||||
self.kv_cache_dtype,
|
||||
prefill.q_data_type,
|
||||
self.kv_lora_rank,
|
||||
self.qk_nope_head_dim,
|
||||
self.qk_rope_head_dim,
|
||||
self.v_head_dim,
|
||||
)
|
||||
and q.dtype == torch.bfloat16
|
||||
and weight is not None
|
||||
and weight.dtype == torch.bfloat16
|
||||
and weight.shape == (self.num_heads * 256, 512)
|
||||
and weight.is_contiguous()
|
||||
and getattr(self.kv_b_proj, "bias", None) is None
|
||||
and getattr(self.kv_b_proj, "weight_scale", None) is None
|
||||
and kv_c_and_k_pe_cache.is_contiguous()
|
||||
and attn_metadata.dcp_context_row_indices is not None
|
||||
)
|
||||
if not eligible:
|
||||
return super()._context_parallel_compute_prefill_context(
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
attn_metadata,
|
||||
k_scale,
|
||||
dcp_world_size,
|
||||
fused_mla_kv_concat_fn,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
assert weight is not None
|
||||
assert attn_metadata.dcp_context_row_indices is not None
|
||||
chunked = prefill.chunked_context
|
||||
assert chunked is not None and chunked.dcp_manager is not None
|
||||
assert prefill.prefill_backend is not None
|
||||
assert len(chunked.chunks) == len(attn_metadata.dcp_context_row_indices)
|
||||
output = output_lse = None
|
||||
for chunk, row_indices in zip(
|
||||
chunked.chunks, attn_metadata.dcp_context_row_indices
|
||||
):
|
||||
gathered = gather_compressed_context(
|
||||
kv_c_and_k_pe_cache,
|
||||
chunked.workspace,
|
||||
prefill.block_table[chunk.request_slice],
|
||||
chunk,
|
||||
chunked.dcp_manager.kv_gather,
|
||||
current_platform.fp8_dtype(),
|
||||
)
|
||||
k, v = expand_context(
|
||||
gathered,
|
||||
k_scale,
|
||||
row_indices,
|
||||
chunk.cu_seq_lens,
|
||||
weight,
|
||||
self.num_heads,
|
||||
self.qk_nope_head_dim,
|
||||
self.qk_rope_head_dim,
|
||||
self.v_head_dim,
|
||||
)
|
||||
attn_output, attn_lse = prefill.prefill_backend.run_prefill_context_chunk(
|
||||
chunk=chunk,
|
||||
q=q[chunk.token_slice],
|
||||
k=k,
|
||||
v=v,
|
||||
)
|
||||
if output is None:
|
||||
if len(chunked.chunks) == 1 and not chunked.empty_token_slices:
|
||||
return attn_output, attn_lse
|
||||
output, output_lse = init_mla_context_partial(
|
||||
chunked,
|
||||
attn_output,
|
||||
attn_lse,
|
||||
num_tokens=q.shape[0],
|
||||
)
|
||||
accumulate_mla_context_chunk(
|
||||
chunk, attn_output, attn_lse, output, output_lse
|
||||
)
|
||||
return output, output_lse
|
||||
|
||||
def _flash_attn_varlen_diff_headdims(
|
||||
self, q, k, v, return_softmax_lse=False, softmax_scale=None, **kwargs
|
||||
):
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compressed DCP context preparation for ROCm MLA."""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonPrefillMetadata
|
||||
|
||||
|
||||
def context_row_indices(
|
||||
chunk: MLACommonPrefillMetadata.ContextChunk, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
"""Map compact request-major rows to the padded rank-major AllGather."""
|
||||
assert chunk.padded_local_seq_lens is not None
|
||||
assert chunk.local_context_lens_allranks is not None
|
||||
assert chunk.local_starts is not None
|
||||
segments = []
|
||||
offset = 0
|
||||
for padded, lengths, start in zip(
|
||||
chunk.padded_local_seq_lens,
|
||||
chunk.local_context_lens_allranks,
|
||||
chunk.local_starts,
|
||||
):
|
||||
for rank, length in enumerate(lengths):
|
||||
count = min(max(0, length - start), padded)
|
||||
if count:
|
||||
first = rank * chunk.num_local_context_tokens + offset
|
||||
segments.append(torch.arange(first, first + count, dtype=torch.int32))
|
||||
offset += padded
|
||||
rows = torch.cat(segments) if segments else torch.empty(0, dtype=torch.int32)
|
||||
assert rows.numel() == chunk.num_context_tokens
|
||||
return rows.to(device=device, non_blocking=True)
|
||||
|
||||
|
||||
def gather_compressed_context(
|
||||
cache: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
chunk: MLACommonPrefillMetadata.ContextChunk,
|
||||
gather: Callable[[torch.Tensor, torch.Tensor], object],
|
||||
cache_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Gather FP8 bytes, preserving the existing workspace row capacity."""
|
||||
assert chunk.local_context_lens_allranks is not None
|
||||
assert workspace.is_contiguous()
|
||||
rows, width = workspace.shape
|
||||
fp8_workspace = workspace.view(cache_dtype).view(-1, width)[:rows]
|
||||
local_rows = chunk.num_local_context_tokens
|
||||
world_size = len(chunk.local_context_lens_allranks[0])
|
||||
offset = rows // (world_size + 1)
|
||||
assert offset * (world_size + 1) == rows
|
||||
assert local_rows <= offset
|
||||
local = fp8_workspace[:local_rows]
|
||||
gathered = fp8_workspace[offset : offset + world_size * local_rows]
|
||||
assert chunk.padded_local_cu_seq_lens is not None
|
||||
ops.cp_gather_cache(
|
||||
cache.view(torch.uint8),
|
||||
local.view(torch.uint8),
|
||||
block_table,
|
||||
chunk.padded_local_cu_seq_lens,
|
||||
chunk.num_requests,
|
||||
chunk.starts,
|
||||
)
|
||||
gather(gathered.view(torch.uint8), local.view(torch.uint8))
|
||||
return gathered
|
||||
|
||||
|
||||
def expand_context(
|
||||
gathered: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
row_indices: torch.Tensor,
|
||||
cu_seq_lens: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fuse row reorganization, dequantization, projection and K/V packing."""
|
||||
k = torch.empty(
|
||||
(row_indices.numel(), num_heads, qk_nope_head_dim + qk_rope_head_dim),
|
||||
device=gathered.device,
|
||||
dtype=weight.dtype,
|
||||
)
|
||||
v = torch.empty(
|
||||
(row_indices.numel(), num_heads, v_head_dim),
|
||||
device=gathered.device,
|
||||
dtype=weight.dtype,
|
||||
)
|
||||
if row_indices.numel():
|
||||
rocm_aiter_ops.gather_kv_b_proj(
|
||||
gathered.unsqueeze(1),
|
||||
scale,
|
||||
cu_seq_lens,
|
||||
row_indices,
|
||||
cu_seq_lens,
|
||||
weight,
|
||||
k,
|
||||
v,
|
||||
)
|
||||
return k, v
|
||||
Reference in New Issue
Block a user