[Kernel][Test] Make kernel tests for mamba dual-HW (CUDA + XPU) (#42736)
Signed-off-by: Dobrzyniewicz, Agata <agata.dobrzyniewicz@intel.com>
This commit is contained in:
@@ -11,9 +11,17 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="causal_conv1d Triton kernels require CUDA-alike or XPU",
|
||||
)
|
||||
|
||||
|
||||
def causal_conv1d_ref(
|
||||
x: torch.Tensor,
|
||||
@@ -149,7 +157,7 @@ def causal_conv1d_opcheck_fn(
|
||||
@pytest.mark.parametrize("width", [4])
|
||||
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
|
||||
def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, itype):
|
||||
device = "cuda"
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 1e-2, 5e-2
|
||||
@@ -196,7 +204,7 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, ity
|
||||
def test_causal_conv1d_update_with_batch_gather(
|
||||
batch_size, with_padding, dim, width, seqlen, has_bias, silu_activation, itype
|
||||
):
|
||||
device = "cuda"
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 1e-2, 5e-2
|
||||
@@ -275,7 +283,7 @@ def test_causal_conv1d_update_with_batch_gather(
|
||||
def test_causal_conv1d_varlen(
|
||||
batch, with_padding, dim, seqlen, width, has_bias, silu_activation, itype
|
||||
):
|
||||
device = "cuda"
|
||||
device = DEVICE
|
||||
torch.accelerator.empty_cache()
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
|
||||
if itype == torch.bfloat16:
|
||||
@@ -341,7 +349,7 @@ def test_causal_conv1d_varlen(
|
||||
weight,
|
||||
bias=bias,
|
||||
conv_states=final_states,
|
||||
query_start_loc=cumsum.cuda(),
|
||||
query_start_loc=cumsum.to(device),
|
||||
cache_indices=padded_state_indices,
|
||||
has_initial_state=has_initial_states,
|
||||
activation=activation,
|
||||
|
||||
@@ -17,6 +17,21 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="mamba_ssm kernels require CUDA-alike or XPU",
|
||||
)
|
||||
|
||||
# selective_scan_fn is backed by the CUDA-only `ops.selective_scan_fwd` C++ op,
|
||||
# so tests exercising it must be skipped on XPU. selective_state_update is
|
||||
# pure Triton and runs on both CUDA-alike and XPU.
|
||||
skip_unless_cuda_alike = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="selective_scan_fn uses CUDA-only custom op",
|
||||
)
|
||||
|
||||
|
||||
def selective_scan_ref(
|
||||
u,
|
||||
@@ -181,6 +196,7 @@ def selective_scan_opcheck_fn(
|
||||
@pytest.mark.parametrize("is_variable_C", [True])
|
||||
@pytest.mark.parametrize("is_variable_B", [True])
|
||||
@pytest.mark.parametrize("scan_chunks", [1, 3])
|
||||
@skip_unless_cuda_alike
|
||||
def test_selective_scan(
|
||||
is_variable_B,
|
||||
is_variable_C,
|
||||
@@ -327,7 +343,7 @@ def test_selective_scan(
|
||||
@pytest.mark.parametrize("dstate", [16, 64])
|
||||
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
|
||||
def test_selective_state_update(dim, dstate, has_z, itype):
|
||||
device = "cuda"
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 1e-2, 5e-2
|
||||
@@ -370,7 +386,7 @@ def test_selective_state_update(dim, dstate, has_z, itype):
|
||||
" on compute capability 10.0 CUDA devices.",
|
||||
)
|
||||
def test_selective_state_update_stochastic_rounding(dim, dstate, has_z, philox_rounds):
|
||||
device = "cuda"
|
||||
device = DEVICE
|
||||
rtol, atol = 5e-3, 1e-1
|
||||
# set seed
|
||||
set_random_seed(0)
|
||||
@@ -417,7 +433,7 @@ def test_selective_state_update_stochastic_rounding(dim, dstate, has_z, philox_r
|
||||
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
|
||||
@pytest.mark.parametrize("max_seq_len", [1, 2, 4])
|
||||
def test_selective_state_update_varlen(dim, dstate, has_z, itype, max_seq_len):
|
||||
device = "cuda"
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 5e-2, 1.5e-1
|
||||
@@ -498,6 +514,7 @@ def test_selective_state_update_varlen(dim, dstate, has_z, itype, max_seq_len):
|
||||
@pytest.mark.parametrize("is_variable_B", [True])
|
||||
# tests correctness in case subset of the sequences are padded
|
||||
@pytest.mark.parametrize("with_padding", [False, True])
|
||||
@skip_unless_cuda_alike
|
||||
def test_selective_scan_varlen(
|
||||
with_padding,
|
||||
is_variable_B,
|
||||
@@ -679,7 +696,7 @@ def test_selective_scan_varlen(
|
||||
def test_selective_state_update_with_batch_indices(
|
||||
with_padding, dim, dstate, has_z, itype
|
||||
):
|
||||
device = "cuda"
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 1e-1, 1e-1
|
||||
@@ -771,7 +788,7 @@ def test_selective_state_update_with_batch_indices(
|
||||
def test_selective_state_update_with_heads_with_batch_indices(
|
||||
dim, dstate, ngroups, has_z, tie_hdim, itype
|
||||
):
|
||||
device = "cuda"
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 3e-2)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 1e-1, 1e-1
|
||||
@@ -844,7 +861,7 @@ def test_selective_state_update_with_heads_with_batch_indices(
|
||||
def test_selective_state_update_with_num_accepted_tokens(
|
||||
dim, dstate, has_z, itype, max_seq_len
|
||||
):
|
||||
device = "cuda"
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 5e-2, 1.5e-1
|
||||
@@ -970,7 +987,7 @@ def test_selective_state_update_with_num_accepted_tokens(
|
||||
def test_selective_state_update_varlen_with_num_accepted(
|
||||
dim, dstate, has_z, itype, max_seq_len
|
||||
):
|
||||
device = "cuda"
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 5e-2, 1.5e-1
|
||||
|
||||
@@ -9,9 +9,19 @@ from einops import rearrange, repeat
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata
|
||||
|
||||
# All kernels exercised here are pure Triton, so they run on any backend
|
||||
# that the vLLM platform layer treats as a CUDA-alike device or as XPU.
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Mamba2 SSD Triton kernels require a CUDA-alike or XPU device.",
|
||||
)
|
||||
|
||||
# Added by the IBM Team, 2024
|
||||
|
||||
# Adapted from https://github.com/state-spaces/mamba/blob/v2.2.4/mamba_ssm/modules/ssd_minimal.py
|
||||
@@ -81,7 +91,7 @@ def ssd_minimal_discrete(X, A, B, C, block_len, initial_states=None):
|
||||
return Y, final_state
|
||||
|
||||
|
||||
def generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype, device="cuda"):
|
||||
def generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype, device=DEVICE):
|
||||
set_random_seed(0)
|
||||
A = -torch.exp(torch.rand(n_heads, dtype=itype, device=device))
|
||||
dt = F.softplus(
|
||||
@@ -103,7 +113,7 @@ def generate_continuous_batched_examples(
|
||||
n_heads,
|
||||
d_head,
|
||||
itype,
|
||||
device="cuda",
|
||||
device=DEVICE,
|
||||
return_naive_ref=True,
|
||||
):
|
||||
# this function generates a random examples of certain length
|
||||
@@ -215,7 +225,7 @@ def test_mamba_chunk_scan_single_example(d_head, n_heads, seq_len_chunk_size, it
|
||||
X * dt.unsqueeze(-1), A * dt, B, C, chunk_size
|
||||
)
|
||||
|
||||
cu_seqlens = torch.tensor((0, seqlen), device="cuda").cumsum(dim=0)
|
||||
cu_seqlens = torch.tensor((0, seqlen), device=DEVICE).cumsum(dim=0)
|
||||
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
|
||||
compute_varlen_chunk_metadata(cu_seqlens, chunk_size)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user