Merge branch 'main' into patches
This commit is contained in:
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -358,3 +360,113 @@ def test_causal_conv1d_varlen(
|
||||
)
|
||||
unpadded_out = out[:, : out_ref_tensor.shape[-1]]
|
||||
assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("itype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("silu_activation", [True])
|
||||
@pytest.mark.parametrize("has_bias", [True])
|
||||
@pytest.mark.parametrize("width", [4])
|
||||
@pytest.mark.parametrize("dim", [64, 2048])
|
||||
@pytest.mark.parametrize("q_lens_cfg", [[3, 1, 0, 4, 2], [1, 5, 5, 1], [2, 0, 0, 3]])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason="CPU kernel for causal_conv1d_update ignores num_accepted_tokens.",
|
||||
)
|
||||
def test_causal_conv1d_update_ragged_varlen(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
):
|
||||
mk_inputs = lambda: _create_ragged_varlen_inputs(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
)
|
||||
inputs, state = mk_inputs()
|
||||
actual = causal_conv1d_update(conv_state=state, **inputs)
|
||||
|
||||
ref_inputs, ref_state = mk_inputs()
|
||||
ref_outs = _create_ragged_varlen_ref_outs(conv_state=ref_state, **ref_inputs)
|
||||
|
||||
for i, ref_out in enumerate(ref_outs):
|
||||
start = int(inputs["query_start_loc"][i])
|
||||
end = int(inputs["query_start_loc"][i + 1])
|
||||
entry = inputs["conv_state_indices"][i : i + 1]
|
||||
assert torch.allclose(actual[start:end], ref_out, rtol=1e-2, atol=5e-2)
|
||||
assert torch.equal(state[entry], ref_state[entry])
|
||||
|
||||
|
||||
def _create_ragged_varlen_inputs(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
):
|
||||
device = DEVICE
|
||||
set_random_seed(0)
|
||||
|
||||
batch_size = len(q_lens_cfg)
|
||||
max_query_len = 5
|
||||
assert max(q_lens_cfg) <= max_query_len
|
||||
state_len = width - 1 + max_query_len
|
||||
total_entries = 10 * batch_size
|
||||
|
||||
q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.cat(
|
||||
[
|
||||
torch.zeros(1, dtype=torch.int32, device=device),
|
||||
torch.cumsum(q_lens, dim=0, dtype=torch.int32),
|
||||
]
|
||||
)
|
||||
total_tokens = int(q_lens.sum())
|
||||
|
||||
x = torch.randn(total_tokens, dim, device=device, dtype=itype)
|
||||
conv_state = torch.randn(
|
||||
total_entries, state_len, dim, device=device, dtype=itype
|
||||
).transpose(1, 2)
|
||||
weight = torch.randn(dim, width, device=device, dtype=itype)
|
||||
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
|
||||
activation = None if not silu_activation else "silu"
|
||||
|
||||
conv_state_indices = (
|
||||
torch.randperm(total_entries - 1, device=device)[:batch_size] + 1
|
||||
).to(torch.int32)
|
||||
num_accepted_tokens = torch.randint(
|
||||
1, max_query_len + 1, (batch_size,), dtype=torch.int32, device=device
|
||||
)
|
||||
inputs = dict(
|
||||
x=x,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
activation=activation,
|
||||
conv_state_indices=conv_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
return inputs, conv_state
|
||||
|
||||
|
||||
def _create_ragged_varlen_ref_outs(
|
||||
x,
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation,
|
||||
conv_state_indices,
|
||||
num_accepted_tokens,
|
||||
query_start_loc,
|
||||
max_query_len,
|
||||
) -> Iterator[torch.Tensor]:
|
||||
for seq_idx in range(len(conv_state_indices)):
|
||||
start = int(query_start_loc[seq_idx])
|
||||
end = int(query_start_loc[seq_idx + 1])
|
||||
entry = conv_state_indices[seq_idx : seq_idx + 1]
|
||||
if start == end:
|
||||
yield x[start:end]
|
||||
continue
|
||||
out_ref = causal_conv1d_update(
|
||||
x[start:end],
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation=activation,
|
||||
conv_state_indices=entry,
|
||||
num_accepted_tokens=num_accepted_tokens[seq_idx : seq_idx + 1],
|
||||
query_start_loc=query_start_loc.new_tensor([0, end - start]),
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
yield out_ref
|
||||
|
||||
@@ -6,6 +6,10 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange, repeat
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_chunk_scan import (
|
||||
_chunk_scan_fwd,
|
||||
_chunk_scan_fwd_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
@@ -578,3 +582,46 @@ def test_mamba_chunk_scan_cont_batch_prefill_chunking(chunk_size, seqlens):
|
||||
rtol=rtol,
|
||||
msg=lambda x, i=i: f"seq{i} state " + x,
|
||||
)
|
||||
|
||||
|
||||
def test_chunk_scan_fwd_every_config_matches(monkeypatch):
|
||||
"""The autotuner picks a config by speed only, so every config in the list
|
||||
must give the same finite output. On sm90, configs that pipeline a wgmma
|
||||
with `cb` in registers wrote NaN into whole rows (#61603)."""
|
||||
from triton.runtime.errors import OutOfResources
|
||||
|
||||
set_random_seed(0)
|
||||
nheads, headdim, ngroups, dstate, chunk_size = 64, 64, 8, 128, 128
|
||||
cu_chunk_seqlens = torch.tensor([0, 128, 181], dtype=torch.int32, device=DEVICE)
|
||||
seqlen, nchunks = 181, 2
|
||||
x = torch.randn(seqlen, nheads, headdim, dtype=torch.bfloat16, device=DEVICE)
|
||||
C = torch.randn(seqlen, ngroups, dstate, dtype=torch.bfloat16, device=DEVICE)
|
||||
cb = torch.randn(nchunks, ngroups, chunk_size, chunk_size, device=DEVICE)
|
||||
dt = torch.rand(nheads, nchunks, chunk_size, device=DEVICE) * 0.1
|
||||
dA_cumsum = -torch.rand(nheads, nchunks, chunk_size, device=DEVICE).cumsum(-1)
|
||||
states = torch.randn(nchunks, nheads, headdim, dstate, device=DEVICE) * 0.1
|
||||
seq_idx = torch.zeros(nchunks, dtype=torch.int32, device=DEVICE)
|
||||
D = torch.randn(nheads, headdim, device=DEVICE) * 0.1
|
||||
|
||||
ref = None
|
||||
for config in list(_chunk_scan_fwd_kernel.configs):
|
||||
monkeypatch.setattr(_chunk_scan_fwd_kernel, "configs", [config])
|
||||
for _ in range(10):
|
||||
out = torch.empty_like(x)
|
||||
try:
|
||||
_chunk_scan_fwd(
|
||||
cb, x, dt, dA_cumsum, C, states, cu_chunk_seqlens, out, seq_idx, D
|
||||
)
|
||||
except OutOfResources:
|
||||
# The autotuner skips configs that do not fit this device.
|
||||
break
|
||||
assert torch.isfinite(out).all(), f"non-finite output with {config}"
|
||||
if ref is None:
|
||||
ref = out
|
||||
torch.testing.assert_close(
|
||||
out,
|
||||
ref,
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
msg=lambda m, config=config: f"{config}: {m}",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user