Merge branch 'main' into patches
This commit is contained in:
@@ -0,0 +1,93 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm Kimi-K3 KDA metadata under adaptive verification.
|
||||
|
||||
Adaptive verification splits the draft budget evenly on the host, then trims
|
||||
each verify request on device. The KDA spec path must therefore plan from the
|
||||
device offsets, never from host lengths that merely look uniform.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import (
|
||||
KimiK3ROCmKDABackend,
|
||||
KimiK3ROCmKDAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
DEVICE = torch.device("cpu")
|
||||
NUM_SPEC = 3
|
||||
|
||||
|
||||
def _config(adaptive: bool):
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=NUM_SPEC,
|
||||
enable_adaptive_verification=adaptive,
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = "none"
|
||||
return vllm_config
|
||||
|
||||
|
||||
def _spec() -> MambaSpec:
|
||||
return MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=NUM_SPEC,
|
||||
)
|
||||
|
||||
|
||||
def _builder(adaptive: bool) -> KimiK3ROCmKDAMetadataBuilder:
|
||||
return KimiK3ROCmKDAMetadataBuilder(
|
||||
kv_cache_spec=_spec(),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=_config(adaptive),
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
|
||||
def _trimmed_verify_batch(builder: KimiK3ROCmKDAMetadataBuilder):
|
||||
"""Host sees [3, 3]; the device was trimmed to [4, 2] (same total)."""
|
||||
batch = BatchSpec(seq_lens=[64, 64], query_lens=[4, 2])
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE, arange_block_indices=True
|
||||
)
|
||||
common = common.replace(
|
||||
query_start_loc_cpu=torch.tensor([0, 3, 6], dtype=torch.int32)
|
||||
)
|
||||
accepted = torch.ones(2, dtype=torch.int32, device=DEVICE)
|
||||
num_decode_draft_tokens_cpu = torch.tensor([NUM_SPEC, NUM_SPEC], dtype=torch.int32)
|
||||
return builder.build(0, common, accepted, num_decode_draft_tokens_cpu)
|
||||
|
||||
|
||||
def test_adaptive_verification_plans_from_device_offsets():
|
||||
"""Equal host lengths would select the fixed-length recurrent kernel,
|
||||
which places sequence i at i * L and ignores the device cu_seqlens."""
|
||||
assert KimiK3ROCmKDABackend.supports_device_cpu_query_lens_mismatch()
|
||||
bound = KimiK3ROCmKDAMetadataBuilder.get_varlen_cudagraph_max_query_len
|
||||
assert bound(_config(adaptive=True), _spec()) == NUM_SPEC + 1
|
||||
assert bound(_config(adaptive=False), _spec()) is None
|
||||
|
||||
meta = _trimmed_verify_batch(_builder(adaptive=True))
|
||||
assert meta.num_spec_decodes == 2
|
||||
assert meta.uniform_spec_sequence_length is None
|
||||
assert torch.equal(
|
||||
meta.spec_query_start_loc.cpu(), torch.tensor([0, 4, 6], dtype=torch.int32)
|
||||
)
|
||||
# Without adaptive verification the host lengths are exact and kept.
|
||||
meta = _trimmed_verify_batch(_builder(adaptive=False))
|
||||
assert meta.uniform_spec_sequence_length == 3
|
||||
@@ -482,7 +482,7 @@ def _spec_for(monkeypatch, **attrs) -> MambaSpec:
|
||||
layer = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention)
|
||||
for name, value in dict(
|
||||
use_prefill_checkpoint=True,
|
||||
use_fused_chunk=True,
|
||||
kda_prefill_backend="fused",
|
||||
num_spec=0,
|
||||
use_safe_gate=True,
|
||||
).items():
|
||||
@@ -506,7 +506,8 @@ def test_opt_in_publishes_chunk_walk_alignment(monkeypatch) -> None:
|
||||
"attr,value",
|
||||
[
|
||||
("use_prefill_checkpoint", False),
|
||||
("use_fused_chunk", False),
|
||||
("kda_prefill_backend", "triton"),
|
||||
("kda_prefill_backend", "gluon"),
|
||||
("use_safe_gate", False),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -77,7 +77,7 @@ def _run(inp: dict, use_fused: bool, out: torch.Tensor | None = None, **kw):
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=inp["cu"],
|
||||
use_fused_chunk=use_fused,
|
||||
backend=kw.pop("backend", "fused" if use_fused else "triton"),
|
||||
out=out,
|
||||
**kw,
|
||||
)
|
||||
@@ -112,6 +112,27 @@ def test_fused_chunk_matches_triton(seqlens: list[int]) -> None:
|
||||
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens",
|
||||
[[1], [64], [1024], [4096], [513, 64, 1, 1200], [512] * 6],
|
||||
)
|
||||
def test_gluon_chunk_matches_triton(seqlens: list[int]) -> None:
|
||||
"""AITER's Gluon chunk_kda must match the Triton chunk path too."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if not rocm_aiter_ops.is_kda_gluon_prefill_available():
|
||||
pytest.skip("AITER Gluon chunk_kda is not available here")
|
||||
|
||||
inp = _inputs(seqlens, seed=len(seqlens))
|
||||
o_ref, state_ref = _run(inp, use_fused=False)
|
||||
o_gluon, state_gluon = _run(inp, use_fused=False, backend="gluon")
|
||||
|
||||
torch.testing.assert_close(o_gluon.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_gluon, state_ref, rtol=5e-3, atol=5e-3)
|
||||
state_rel = (state_gluon - state_ref).norm() / state_ref.norm()
|
||||
assert state_rel < 5e-3, f"final state relative L2 error {state_rel:.2e}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_out_argument_receives_the_result(use_fused: bool) -> None:
|
||||
"""``out`` is a promise about where the result lands.
|
||||
@@ -648,7 +669,7 @@ def _run_paged(inp: dict, cache, idx, warm, use_fused: bool, **kw):
|
||||
has_initial_state=warm,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=inp["cu"],
|
||||
use_fused_chunk=use_fused,
|
||||
backend="fused" if use_fused else "triton",
|
||||
**kw,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="AITER fused conv1d is ROCm-only"
|
||||
)
|
||||
|
||||
WIDTH = 4 # Kimi-K3 short_conv_kernel_size
|
||||
STATE_LEN = WIDTH - 1
|
||||
HEAD_DIM = 128
|
||||
LOCAL_HEADS = 4 # per-TP-rank heads; projection = 512, packed dim = 1536
|
||||
|
||||
|
||||
def _fused_fn():
|
||||
gdn = pytest.importorskip("aiter.ops.triton.gated_delta_net")
|
||||
return gdn.causal_conv1d_split_qkv_triton_tile_fn
|
||||
|
||||
|
||||
def _make_inputs(seqlens, has_initial_state, seed=0, device="cuda"):
|
||||
torch.manual_seed(seed)
|
||||
proj = HEAD_DIM * LOCAL_HEADS
|
||||
dim = 3 * proj
|
||||
total = sum(seqlens)
|
||||
n_seqs = len(seqlens)
|
||||
# Token-major activations, as produced by the in_proj GEMM.
|
||||
mixed_qkv = torch.randn(total, dim, dtype=torch.bfloat16, device=device)
|
||||
weight = torch.randn(dim, WIDTH, dtype=torch.bfloat16, device=device) * 0.3
|
||||
# More cache lines than sequences, assigned in a shuffled order. Line 0 is
|
||||
# vLLM's null block (NULL_BLOCK_ID), which causal_conv1d_fn skips, so real
|
||||
# sequences never map to it.
|
||||
n_lines = n_seqs + 2
|
||||
conv_state = torch.randn(
|
||||
n_lines, dim, STATE_LEN, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
cache_indices = (torch.randperm(n_lines - 1, device=device)[:n_seqs] + 1).to(
|
||||
torch.int32
|
||||
)
|
||||
query_start_loc = torch.tensor(
|
||||
[0] + list(torch.tensor(seqlens).cumsum(0).tolist()),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
has_initial = torch.full(
|
||||
(n_seqs,), has_initial_state, dtype=torch.bool, device=device
|
||||
)
|
||||
return mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
|
||||
|
||||
|
||||
def _reference(
|
||||
mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
|
||||
):
|
||||
"""fp32 per-sequence depthwise causal conv + SiLU.
|
||||
|
||||
Returns (out[T, dim], new_conv_state).
|
||||
"""
|
||||
x = mixed_qkv.float().t() # (dim, T)
|
||||
w = weight.float()
|
||||
dim = x.shape[0]
|
||||
out = torch.empty_like(x)
|
||||
new_state = conv_state.float().clone()
|
||||
qsl = query_start_loc.tolist()
|
||||
for s in range(len(qsl) - 1):
|
||||
start, end = qsl[s], qsl[s + 1]
|
||||
ci = int(cache_indices[s])
|
||||
xs = x[:, start:end]
|
||||
if bool(has_initial[s]):
|
||||
xin = torch.cat([conv_state[ci].float(), xs], dim=1)
|
||||
else:
|
||||
xin = F.pad(xs, (STATE_LEN, 0))
|
||||
y = F.conv1d(xin.unsqueeze(0), w.unsqueeze(1), None, groups=dim)[0]
|
||||
out[:, start:end] = F.silu(y)
|
||||
new_state[ci] = xin[:, -STATE_LEN:]
|
||||
return out.t(), new_state
|
||||
|
||||
|
||||
def _three_launch(
|
||||
mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
|
||||
):
|
||||
"""The pre-fusion KDA prefill path: one causal_conv1d_fn per projection."""
|
||||
proj = mixed_qkv.shape[1] // 3
|
||||
outs = []
|
||||
for x, state, w in zip(
|
||||
mixed_qkv.split(proj, dim=-1),
|
||||
conv_state.split(proj, dim=-2),
|
||||
weight.split(proj, dim=0),
|
||||
):
|
||||
outs.append(
|
||||
causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
w,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial,
|
||||
cache_indices=cache_indices,
|
||||
query_start_loc=query_start_loc,
|
||||
).transpose(0, 1)
|
||||
)
|
||||
return outs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens", [[1024], [8, 1, 300], [64, 64, 64], [513, 64, 1, 1200]]
|
||||
)
|
||||
@pytest.mark.parametrize("has_initial_state", [False, True])
|
||||
@pytest.mark.parametrize("block_m", [8, 128])
|
||||
def test_fused_split_qkv_conv1d_matches_reference(seqlens, has_initial_state, block_m):
|
||||
fused_fn = _fused_fn()
|
||||
mixed_qkv, weight, conv_state, cache_indices, qsl, has_initial = _make_inputs(
|
||||
seqlens, has_initial_state
|
||||
)
|
||||
proj = mixed_qkv.shape[1] // 3
|
||||
|
||||
ref_out, ref_state = _reference(
|
||||
mixed_qkv, weight, conv_state, cache_indices, qsl, has_initial
|
||||
)
|
||||
|
||||
state_3 = conv_state.clone()
|
||||
q3, k3, v3 = _three_launch(
|
||||
mixed_qkv, weight, state_3, cache_indices, qsl, has_initial
|
||||
)
|
||||
|
||||
state_f = conv_state.clone()
|
||||
# Exactly the call made by KimiK3DeltaAttention._forward.
|
||||
qf, kf, vf = fused_fn(
|
||||
mixed_qkv.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
state_f,
|
||||
qsl,
|
||||
proj,
|
||||
proj,
|
||||
cache_indices=cache_indices,
|
||||
has_initial_state=has_initial,
|
||||
activation="silu",
|
||||
block_m=block_m,
|
||||
metadata=None,
|
||||
preserve_input_dtype=True,
|
||||
)
|
||||
|
||||
fused = torch.cat([qf, kf, vf], dim=-1)
|
||||
assert fused.shape == mixed_qkv.shape and fused.dtype == mixed_qkv.dtype
|
||||
assert all(t.is_contiguous() for t in (qf, kf, vf))
|
||||
|
||||
tol = dict(atol=2e-2, rtol=2e-2)
|
||||
torch.testing.assert_close(fused.float(), ref_out, **tol)
|
||||
torch.testing.assert_close(fused, torch.cat([q3, k3, v3], dim=-1), **tol)
|
||||
|
||||
# conv_state writeback: assigned lines updated, the others untouched.
|
||||
torch.testing.assert_close(state_f.float(), ref_state, **tol)
|
||||
torch.testing.assert_close(state_f, state_3, **tol)
|
||||
untouched = torch.ones(conv_state.shape[0], dtype=torch.bool, device="cuda")
|
||||
untouched[cache_indices.long()] = False
|
||||
assert untouched[NULL_BLOCK_ID]
|
||||
assert torch.equal(state_f[untouched], conv_state[untouched])
|
||||
Reference in New Issue
Block a user