[Hardware][CPU] Enable granite-4 model on cpu (#47641)
Signed-off-by: Akash Kaothalkar <akashkaothalkar@akashs-mbp.bl1-in.ibm.com> Signed-off-by: Akash Kaothalkar <akashkaothalkar@dhcp-9-123-5-76.bl1-in.ibm.com> Signed-off-by: Akash Kaothalkar <akashkaothalkar@Akashs-MBP.lan> Signed-off-by: Akash kaothalkar <akash.kaothalkar@ibm.com> Co-authored-by: Akash Kaothalkar <akashkaothalkar@dhcp-9-123-5-76.bl1-in.ibm.com> Co-authored-by: Akash Kaothalkar <akashkaothalkar@Akashs-MBP.lan> Co-authored-by: Akash Kaothalkar <akashkaothalkar@akashs-mbp.bl1-in.ibm.com> Co-authored-by: Akash kaothalkar <akash.kaothalkar@ibm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Akash Kaothalkar
Akash Kaothalkar
Akash Kaothalkar
Akash kaothalkar
Li, Jiang <jiang1.li@intel.com>
parent
dcfebf93f4
commit
5c9f6557d7
@@ -425,7 +425,7 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No
|
||||
match the single-call result.
|
||||
"""
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_torch,
|
||||
causal_conv1d_fn_cpu as causal_conv1d_torch,
|
||||
)
|
||||
|
||||
x, weight, bias = _conv_inputs(total_tokens)
|
||||
|
||||
@@ -18,8 +18,12 @@ from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="causal_conv1d Triton kernels require CUDA-alike or XPU",
|
||||
not (
|
||||
current_platform.is_cuda_alike()
|
||||
or current_platform.is_xpu()
|
||||
or current_platform.is_cpu()
|
||||
),
|
||||
reason="causal_conv1d Triton kernels require CUDA-alike, XPU, or CPU",
|
||||
)
|
||||
|
||||
|
||||
@@ -284,7 +288,8 @@ def test_causal_conv1d_varlen(
|
||||
batch, with_padding, dim, seqlen, width, has_bias, silu_activation, itype
|
||||
):
|
||||
device = DEVICE
|
||||
torch.accelerator.empty_cache()
|
||||
if not current_platform.is_cpu():
|
||||
torch.accelerator.empty_cache()
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 1e-2, 5e-2
|
||||
|
||||
@@ -20,8 +20,12 @@ from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="mamba_ssm kernels require CUDA-alike or XPU",
|
||||
not (
|
||||
current_platform.is_cuda_alike()
|
||||
or current_platform.is_xpu()
|
||||
or current_platform.is_cpu()
|
||||
),
|
||||
reason="mamba_ssm kernels require CUDA-alike, XPU, or CPU",
|
||||
)
|
||||
|
||||
# selective_scan_fn is backed by the CUDA-only `ops.selective_scan_fwd` C++ op,
|
||||
@@ -342,6 +346,13 @@ def test_selective_scan(
|
||||
@pytest.mark.parametrize("has_z", [False, True])
|
||||
@pytest.mark.parametrize("dstate", [16, 64])
|
||||
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason=(
|
||||
"CPU kernel for selective_state_update only supports "
|
||||
"Mamba 2 (scalar A/dt), not Mamba 1."
|
||||
),
|
||||
)
|
||||
def test_selective_state_update(dim, dstate, has_z, itype):
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
|
||||
@@ -436,6 +447,13 @@ def test_selective_state_update_stochastic_rounding(dim, dstate, has_z, philox_r
|
||||
@pytest.mark.parametrize("dstate", [16, 64])
|
||||
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
|
||||
@pytest.mark.parametrize("max_seq_len", [1, 2, 4])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason=(
|
||||
"CPU kernel for selective_state_update only supports "
|
||||
"Mamba 2 (scalar A/dt), not Mamba 1."
|
||||
),
|
||||
)
|
||||
def test_selective_state_update_varlen(dim, dstate, has_z, itype, max_seq_len):
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
|
||||
@@ -697,6 +715,13 @@ def test_selective_scan_varlen(
|
||||
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
|
||||
# tests correctness in case subset of the sequences are padded
|
||||
@pytest.mark.parametrize("with_padding", [True, False])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason=(
|
||||
"CPU kernel for selective_state_update only supports "
|
||||
"Mamba 2 (scalar A/dt), not Mamba 1."
|
||||
),
|
||||
)
|
||||
def test_selective_state_update_with_batch_indices(
|
||||
with_padding, dim, dstate, has_z, itype
|
||||
):
|
||||
@@ -789,6 +814,13 @@ def test_selective_state_update_with_batch_indices(
|
||||
@pytest.mark.parametrize("ngroups", [1, 4])
|
||||
@pytest.mark.parametrize("dstate", [16, 64])
|
||||
@pytest.mark.parametrize("dim", [2048, 4096])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason=(
|
||||
"CPU kernel for selective_state_update only supports "
|
||||
"Mamba 2 (scalar A/dt), not Mamba 1."
|
||||
),
|
||||
)
|
||||
def test_selective_state_update_with_heads_with_batch_indices(
|
||||
dim, dstate, ngroups, has_z, tie_hdim, itype
|
||||
):
|
||||
@@ -862,6 +894,13 @@ def test_selective_state_update_with_heads_with_batch_indices(
|
||||
@pytest.mark.parametrize("dstate", [16, 64])
|
||||
@pytest.mark.parametrize("dim", [2048, 4096])
|
||||
@pytest.mark.parametrize("max_seq_len", [2, 4])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason=(
|
||||
"CPU kernel for selective_state_update only supports "
|
||||
"Mamba 2 (scalar A/dt), not Mamba 1."
|
||||
),
|
||||
)
|
||||
def test_selective_state_update_with_num_accepted_tokens(
|
||||
dim, dstate, has_z, itype, max_seq_len
|
||||
):
|
||||
@@ -988,6 +1027,13 @@ def test_selective_state_update_with_num_accepted_tokens(
|
||||
@pytest.mark.parametrize("dstate", [16, 64])
|
||||
@pytest.mark.parametrize("dim", [2048, 4096])
|
||||
@pytest.mark.parametrize("max_seq_len", [2, 4])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason=(
|
||||
"CPU kernel for selective_state_update only supports "
|
||||
"Mamba 2 (scalar A/dt), not Mamba 1."
|
||||
),
|
||||
)
|
||||
def test_selective_state_update_varlen_with_num_accepted(
|
||||
dim, dstate, has_z, itype, max_seq_len
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user