[CPU] Support FP16/BF16 persisted GDN state on AMX (#52191)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
2a61f060d3
commit
c6c33f2b1f
@@ -37,6 +37,7 @@ CHUNK_HEAD_DIMS = [
|
||||
CHUNK_SIZE = 64
|
||||
CONV_DIM = 128
|
||||
CONV_KERNEL = 4
|
||||
DECODE_BATCH_SIZES = [1, 3, 5]
|
||||
PREFILL_SEQ_LENS = [
|
||||
[1],
|
||||
[1, 2, 3],
|
||||
@@ -47,7 +48,36 @@ PREFILL_SEQ_LENS = [
|
||||
[2 * CHUNK_SIZE - 1, 2 * CHUNK_SIZE, 2 * CHUNK_SIZE + 1],
|
||||
[4 * CHUNK_SIZE + 17],
|
||||
]
|
||||
DECODE_BATCH_SIZES = [1, 3, 5]
|
||||
DECODE_DTYPE_CASES = [
|
||||
*[
|
||||
pytest.param(
|
||||
batch_size,
|
||||
num_heads,
|
||||
head_dims,
|
||||
torch.float32,
|
||||
id=f"fp32-b{batch_size}-h{num_heads}-d{head_dims}",
|
||||
)
|
||||
for batch_size in [1, 3, 5]
|
||||
for num_heads in NUM_HEADS
|
||||
for head_dims in HEAD_DIMS
|
||||
],
|
||||
pytest.param(3, (2, 4), (32, 32), torch.bfloat16, id="bf16-representative"),
|
||||
pytest.param(3, (2, 4), (32, 32), torch.float16, id="fp16-representative"),
|
||||
]
|
||||
PREFILL_DTYPE_CASES = [
|
||||
*[
|
||||
pytest.param(
|
||||
num_heads,
|
||||
head_dims,
|
||||
torch.float32,
|
||||
id=f"fp32-h{num_heads}-d{head_dims}",
|
||||
)
|
||||
for num_heads in NUM_HEADS
|
||||
for head_dims in CHUNK_HEAD_DIMS
|
||||
],
|
||||
pytest.param((2, 4), (64, 64), torch.bfloat16, id="bf16-representative"),
|
||||
pytest.param((2, 4), (64, 64), torch.float16, id="fp16-representative"),
|
||||
]
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=128, typed=False)
|
||||
@@ -91,6 +121,8 @@ def ref_gated_delta_rule(
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
state_dtype: torch.dtype = torch.float32,
|
||||
state_write_interval: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
g, beta = ref_gdn_gating(A_log, a, b, dt_bias)
|
||||
out = torch.empty_like(value)
|
||||
@@ -144,6 +176,10 @@ def ref_gated_delta_rule(
|
||||
delta = (v_t - kv_mem) * beta_t
|
||||
state = state + delta.unsqueeze(-1) * k_t.unsqueeze(-2)
|
||||
out_seq[:, :, token_idx] = (state * q_t.unsqueeze(-2)).sum(dim=-1)
|
||||
if state_dtype != torch.float32 and (
|
||||
(token_idx + 1) % state_write_interval == 0 or token_idx + 1 == seq_len
|
||||
):
|
||||
state = state.to(state_dtype).float()
|
||||
|
||||
out[:, begin:end] = out_seq.transpose(1, 2).contiguous().to(initial_dtype)
|
||||
final_state[seq_idx] = state.squeeze(0)
|
||||
@@ -201,14 +237,15 @@ def test_fused_gdn_gating_cpu(
|
||||
|
||||
|
||||
# decode path
|
||||
@pytest.mark.parametrize("batch_size", DECODE_BATCH_SIZES)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_dims", HEAD_DIMS)
|
||||
@pytest.mark.parametrize(
|
||||
"batch_size,num_heads,head_dims,state_dtype", DECODE_DTYPE_CASES
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
batch_size: int,
|
||||
num_heads: tuple[int, int],
|
||||
head_dims: tuple[int, int],
|
||||
state_dtype: torch.dtype,
|
||||
) -> None:
|
||||
q, k, v, a, b, A_log, dt_bias = gdn_inputs(
|
||||
num_tokens=batch_size,
|
||||
@@ -221,7 +258,7 @@ def test_fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
cu_seqlens = torch.arange(batch_size + 1, dtype=torch.int32)
|
||||
state_shape = (batch_size, num_v_heads, head_dim, v_head_dim)
|
||||
state = tensor_cache(
|
||||
batch_size * num_v_heads * head_dim * v_head_dim, torch.float32
|
||||
batch_size * num_v_heads * head_dim * v_head_dim, state_dtype
|
||||
).view(state_shape)
|
||||
state_ref = state[state_indices].transpose(-1, -2).contiguous()
|
||||
|
||||
@@ -236,6 +273,7 @@ def test_fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
initial_state=state_ref,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
state_dtype=state_dtype,
|
||||
)
|
||||
out_ref = out_ref.transpose(0, 1).contiguous()
|
||||
|
||||
@@ -256,8 +294,8 @@ def test_fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
|
||||
torch.testing.assert_close(out, out_ref, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(
|
||||
state_out[state_indices].transpose(-1, -2),
|
||||
final_state_ref,
|
||||
state_out[state_indices].transpose(-1, -2).float(),
|
||||
final_state_ref.float(),
|
||||
atol=1e-2,
|
||||
rtol=1e-2,
|
||||
)
|
||||
@@ -265,13 +303,13 @@ def test_fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
|
||||
# prefill path
|
||||
@pytest.mark.parametrize("seq_lens", PREFILL_SEQ_LENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_dims", CHUNK_HEAD_DIMS)
|
||||
@pytest.mark.parametrize("num_heads,head_dims,state_dtype", PREFILL_DTYPE_CASES)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_gated_delta_rule_cpu(
|
||||
seq_lens: list[int],
|
||||
num_heads: tuple[int, int],
|
||||
head_dims: tuple[int, int],
|
||||
state_dtype: torch.dtype,
|
||||
) -> None:
|
||||
total_tokens = sum(seq_lens)
|
||||
q, k, v, a, b, A_log, dt_bias = gdn_inputs(
|
||||
@@ -286,7 +324,7 @@ def test_chunk_gated_delta_rule_cpu(
|
||||
)
|
||||
initial_state_shape = (len(seq_lens), num_v_heads, head_dim, v_head_dim)
|
||||
initial_state = tensor_cache(
|
||||
len(seq_lens) * num_v_heads * head_dim * v_head_dim, torch.float32
|
||||
len(seq_lens) * num_v_heads * head_dim * v_head_dim, state_dtype
|
||||
).view(initial_state_shape)
|
||||
initial_state_ref = initial_state.transpose(-1, -2).contiguous()
|
||||
|
||||
@@ -301,6 +339,8 @@ def test_chunk_gated_delta_rule_cpu(
|
||||
initial_state=initial_state_ref,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
state_dtype=state_dtype,
|
||||
state_write_interval=CHUNK_SIZE,
|
||||
)
|
||||
|
||||
g, beta = ref_gdn_gating(A_log, a, b, dt_bias)
|
||||
@@ -320,8 +360,8 @@ def test_chunk_gated_delta_rule_cpu(
|
||||
|
||||
torch.testing.assert_close(out, out_ref, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(
|
||||
final_state.transpose(-1, -2),
|
||||
final_state_ref,
|
||||
final_state.transpose(-1, -2).float(),
|
||||
final_state_ref.float(),
|
||||
atol=1e-2,
|
||||
rtol=1e-2,
|
||||
)
|
||||
@@ -339,14 +379,16 @@ TWO_CALL_SPLITS = [
|
||||
|
||||
|
||||
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_dims", CHUNK_HEAD_DIMS)
|
||||
@pytest.mark.parametrize("num_heads", [(2, 4)])
|
||||
@pytest.mark.parametrize("head_dims", [(64, 64)])
|
||||
@pytest.mark.parametrize("state_dtype", [torch.float32, torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_chunk_gated_delta_rule_cpu_two_call_split(
|
||||
total_tokens: int,
|
||||
split: int,
|
||||
num_heads: tuple[int, int],
|
||||
head_dims: tuple[int, int],
|
||||
state_dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""A prefill split into two calls (the second seeded with the first's
|
||||
``final_state`` and a rebased ``cu_seqlens``) must match the single-call
|
||||
@@ -365,7 +407,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
|
||||
g = g.unsqueeze(0) # [1, T, HV]
|
||||
beta = beta.unsqueeze(0)
|
||||
|
||||
zero_state = torch.zeros(1, num_v_heads, head_dim, v_head_dim, dtype=torch.float32)
|
||||
zero_state = torch.zeros(1, num_v_heads, head_dim, v_head_dim, dtype=state_dtype)
|
||||
|
||||
# Reference: whole sequence in one call, no initial state.
|
||||
out_full, final_full = ops.chunk_gated_delta_rule_cpu(
|
||||
@@ -405,7 +447,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
|
||||
value=v[:, split:],
|
||||
g=g[:, split:],
|
||||
beta=beta[:, split:],
|
||||
initial_state=state1.to(torch.float32),
|
||||
initial_state=state1,
|
||||
output_final_state=True,
|
||||
cu_seqlens=torch.tensor([0, tail], dtype=torch.int32),
|
||||
head_first=False,
|
||||
@@ -416,7 +458,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
|
||||
out_split = torch.cat([out1, out2], dim=1)
|
||||
|
||||
# State must be near-exact; output allows a looser bound for the bf16 round-trip.
|
||||
torch.testing.assert_close(state2, final_full, atol=1e-3, rtol=1e-3)
|
||||
torch.testing.assert_close(state2.float(), final_full.float(), atol=1e-3, rtol=1e-3)
|
||||
torch.testing.assert_close(out_split, out_full, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
|
||||
|
||||
class _FakeModel:
|
||||
@classmethod
|
||||
def get_mamba_state_dtype_from_config(
|
||||
cls, vllm_config: VllmConfig
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
state_dtype = {
|
||||
"float16": torch.float16,
|
||||
"bfloat16": torch.bfloat16,
|
||||
}.get(vllm_config.cache_config.mamba_ssm_cache_dtype, torch.float32)
|
||||
return torch.float32, state_dtype
|
||||
|
||||
|
||||
class _FakeRegistry:
|
||||
@staticmethod
|
||||
def resolve_model_cls(
|
||||
architecture: str,
|
||||
model_config: SimpleNamespace,
|
||||
) -> tuple[type[_FakeModel], str]:
|
||||
return _FakeModel, architecture
|
||||
|
||||
|
||||
def _cpu_config(
|
||||
cache_config: CacheConfig,
|
||||
*,
|
||||
model_type: str,
|
||||
resolved_dtype: str,
|
||||
architecture: str | None = None,
|
||||
layer_types: tuple[str, ...] = ("linear_attention",),
|
||||
) -> SimpleNamespace:
|
||||
model_config = SimpleNamespace(
|
||||
disable_cascade_attn=False,
|
||||
architecture=architecture or model_type,
|
||||
has_inner_state=True,
|
||||
use_mla=False,
|
||||
hf_text_config=SimpleNamespace(
|
||||
model_type=model_type,
|
||||
layer_types=layer_types,
|
||||
),
|
||||
registry=_FakeRegistry,
|
||||
)
|
||||
config = VllmConfig(cache_config=cache_config)
|
||||
config.model_config = model_config
|
||||
cache_config.mamba_ssm_cache_dtype = resolved_dtype
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"model_type",
|
||||
"requested_dtype",
|
||||
"resolved_dtype",
|
||||
"layer_types",
|
||||
"expected_dtype",
|
||||
),
|
||||
[
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"bfloat16",
|
||||
"bfloat16",
|
||||
("linear_attention",),
|
||||
"bfloat16",
|
||||
id="gdn-explicit-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"float16",
|
||||
"float16",
|
||||
("linear_attention",),
|
||||
"float16",
|
||||
id="gdn-explicit-fp16",
|
||||
),
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"auto",
|
||||
"bfloat16",
|
||||
("linear_attention",),
|
||||
"bfloat16",
|
||||
id="gdn-model-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"auto",
|
||||
"float16",
|
||||
("linear_attention",),
|
||||
"float16",
|
||||
id="gdn-model-fp16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"auto",
|
||||
"bfloat16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-model-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"auto",
|
||||
"float16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-model-fp16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"bfloat16",
|
||||
"bfloat16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-explicit-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"float16",
|
||||
"float16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-explicit-fp16",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_cpu_accelerated_gdn_dtype_policy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_type: str,
|
||||
requested_dtype: str,
|
||||
resolved_dtype: str,
|
||||
layer_types: tuple[str, ...],
|
||||
expected_dtype: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr("torch.cpu._is_avx512_bf16_supported", lambda: True)
|
||||
cache_config = CacheConfig(mamba_ssm_cache_dtype=requested_dtype)
|
||||
config = _cpu_config(
|
||||
cache_config,
|
||||
model_type=model_type,
|
||||
resolved_dtype=resolved_dtype,
|
||||
layer_types=layer_types,
|
||||
)
|
||||
|
||||
CpuPlatform.check_and_update_config(config)
|
||||
assert cache_config.mamba_ssm_cache_dtype == expected_dtype
|
||||
Reference in New Issue
Block a user