[CPU] Support FP16/BF16 persisted GDN state on AMX (#52191)

Signed-off-by: Li, Tianmu <tianmu.li@intel.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Tianmu Li
2026-08-31 16:32:20 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 2a61f060d3
commit c6c33f2b1f
4 changed files with 575 additions and 141 deletions
+59 -17
View File
@@ -37,6 +37,7 @@ CHUNK_HEAD_DIMS = [
CHUNK_SIZE = 64
CONV_DIM = 128
CONV_KERNEL = 4
DECODE_BATCH_SIZES = [1, 3, 5]
PREFILL_SEQ_LENS = [
[1],
[1, 2, 3],
@@ -47,7 +48,36 @@ PREFILL_SEQ_LENS = [
[2 * CHUNK_SIZE - 1, 2 * CHUNK_SIZE, 2 * CHUNK_SIZE + 1],
[4 * CHUNK_SIZE + 17],
]
DECODE_BATCH_SIZES = [1, 3, 5]
DECODE_DTYPE_CASES = [
*[
pytest.param(
batch_size,
num_heads,
head_dims,
torch.float32,
id=f"fp32-b{batch_size}-h{num_heads}-d{head_dims}",
)
for batch_size in [1, 3, 5]
for num_heads in NUM_HEADS
for head_dims in HEAD_DIMS
],
pytest.param(3, (2, 4), (32, 32), torch.bfloat16, id="bf16-representative"),
pytest.param(3, (2, 4), (32, 32), torch.float16, id="fp16-representative"),
]
PREFILL_DTYPE_CASES = [
*[
pytest.param(
num_heads,
head_dims,
torch.float32,
id=f"fp32-h{num_heads}-d{head_dims}",
)
for num_heads in NUM_HEADS
for head_dims in CHUNK_HEAD_DIMS
],
pytest.param((2, 4), (64, 64), torch.bfloat16, id="bf16-representative"),
pytest.param((2, 4), (64, 64), torch.float16, id="fp16-representative"),
]
@functools.lru_cache(maxsize=128, typed=False)
@@ -91,6 +121,8 @@ def ref_gated_delta_rule(
initial_state: torch.Tensor,
cu_seqlens: torch.Tensor,
use_qk_l2norm_in_kernel: bool = False,
state_dtype: torch.dtype = torch.float32,
state_write_interval: int = 1,
) -> tuple[torch.Tensor, torch.Tensor]:
g, beta = ref_gdn_gating(A_log, a, b, dt_bias)
out = torch.empty_like(value)
@@ -144,6 +176,10 @@ def ref_gated_delta_rule(
delta = (v_t - kv_mem) * beta_t
state = state + delta.unsqueeze(-1) * k_t.unsqueeze(-2)
out_seq[:, :, token_idx] = (state * q_t.unsqueeze(-2)).sum(dim=-1)
if state_dtype != torch.float32 and (
(token_idx + 1) % state_write_interval == 0 or token_idx + 1 == seq_len
):
state = state.to(state_dtype).float()
out[:, begin:end] = out_seq.transpose(1, 2).contiguous().to(initial_dtype)
final_state[seq_idx] = state.squeeze(0)
@@ -201,14 +237,15 @@ def test_fused_gdn_gating_cpu(
# decode path
@pytest.mark.parametrize("batch_size", DECODE_BATCH_SIZES)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_dims", HEAD_DIMS)
@pytest.mark.parametrize(
"batch_size,num_heads,head_dims,state_dtype", DECODE_DTYPE_CASES
)
@torch.inference_mode()
def test_fused_sigmoid_gating_delta_rule_update_cpu(
batch_size: int,
num_heads: tuple[int, int],
head_dims: tuple[int, int],
state_dtype: torch.dtype,
) -> None:
q, k, v, a, b, A_log, dt_bias = gdn_inputs(
num_tokens=batch_size,
@@ -221,7 +258,7 @@ def test_fused_sigmoid_gating_delta_rule_update_cpu(
cu_seqlens = torch.arange(batch_size + 1, dtype=torch.int32)
state_shape = (batch_size, num_v_heads, head_dim, v_head_dim)
state = tensor_cache(
batch_size * num_v_heads * head_dim * v_head_dim, torch.float32
batch_size * num_v_heads * head_dim * v_head_dim, state_dtype
).view(state_shape)
state_ref = state[state_indices].transpose(-1, -2).contiguous()
@@ -236,6 +273,7 @@ def test_fused_sigmoid_gating_delta_rule_update_cpu(
initial_state=state_ref,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=True,
state_dtype=state_dtype,
)
out_ref = out_ref.transpose(0, 1).contiguous()
@@ -256,8 +294,8 @@ def test_fused_sigmoid_gating_delta_rule_update_cpu(
torch.testing.assert_close(out, out_ref, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(
state_out[state_indices].transpose(-1, -2),
final_state_ref,
state_out[state_indices].transpose(-1, -2).float(),
final_state_ref.float(),
atol=1e-2,
rtol=1e-2,
)
@@ -265,13 +303,13 @@ def test_fused_sigmoid_gating_delta_rule_update_cpu(
# prefill path
@pytest.mark.parametrize("seq_lens", PREFILL_SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_dims", CHUNK_HEAD_DIMS)
@pytest.mark.parametrize("num_heads,head_dims,state_dtype", PREFILL_DTYPE_CASES)
@torch.inference_mode()
def test_chunk_gated_delta_rule_cpu(
seq_lens: list[int],
num_heads: tuple[int, int],
head_dims: tuple[int, int],
state_dtype: torch.dtype,
) -> None:
total_tokens = sum(seq_lens)
q, k, v, a, b, A_log, dt_bias = gdn_inputs(
@@ -286,7 +324,7 @@ def test_chunk_gated_delta_rule_cpu(
)
initial_state_shape = (len(seq_lens), num_v_heads, head_dim, v_head_dim)
initial_state = tensor_cache(
len(seq_lens) * num_v_heads * head_dim * v_head_dim, torch.float32
len(seq_lens) * num_v_heads * head_dim * v_head_dim, state_dtype
).view(initial_state_shape)
initial_state_ref = initial_state.transpose(-1, -2).contiguous()
@@ -301,6 +339,8 @@ def test_chunk_gated_delta_rule_cpu(
initial_state=initial_state_ref,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=True,
state_dtype=state_dtype,
state_write_interval=CHUNK_SIZE,
)
g, beta = ref_gdn_gating(A_log, a, b, dt_bias)
@@ -320,8 +360,8 @@ def test_chunk_gated_delta_rule_cpu(
torch.testing.assert_close(out, out_ref, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(
final_state.transpose(-1, -2),
final_state_ref,
final_state.transpose(-1, -2).float(),
final_state_ref.float(),
atol=1e-2,
rtol=1e-2,
)
@@ -339,14 +379,16 @@ TWO_CALL_SPLITS = [
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_dims", CHUNK_HEAD_DIMS)
@pytest.mark.parametrize("num_heads", [(2, 4)])
@pytest.mark.parametrize("head_dims", [(64, 64)])
@pytest.mark.parametrize("state_dtype", [torch.float32, torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_chunk_gated_delta_rule_cpu_two_call_split(
total_tokens: int,
split: int,
num_heads: tuple[int, int],
head_dims: tuple[int, int],
state_dtype: torch.dtype,
) -> None:
"""A prefill split into two calls (the second seeded with the first's
``final_state`` and a rebased ``cu_seqlens``) must match the single-call
@@ -365,7 +407,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
g = g.unsqueeze(0) # [1, T, HV]
beta = beta.unsqueeze(0)
zero_state = torch.zeros(1, num_v_heads, head_dim, v_head_dim, dtype=torch.float32)
zero_state = torch.zeros(1, num_v_heads, head_dim, v_head_dim, dtype=state_dtype)
# Reference: whole sequence in one call, no initial state.
out_full, final_full = ops.chunk_gated_delta_rule_cpu(
@@ -405,7 +447,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
value=v[:, split:],
g=g[:, split:],
beta=beta[:, split:],
initial_state=state1.to(torch.float32),
initial_state=state1,
output_final_state=True,
cu_seqlens=torch.tensor([0, tail], dtype=torch.int32),
head_first=False,
@@ -416,7 +458,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
out_split = torch.cat([out1, out2], dim=1)
# State must be near-exact; output allows a looser bound for the bf16 round-trip.
torch.testing.assert_close(state2, final_full, atol=1e-3, rtol=1e-3)
torch.testing.assert_close(state2.float(), final_full.float(), atol=1e-3, rtol=1e-3)
torch.testing.assert_close(out_split, out_full, atol=2e-2, rtol=2e-2)
+152
View File
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.config import CacheConfig, VllmConfig
from vllm.platforms.cpu import CpuPlatform
class _FakeModel:
@classmethod
def get_mamba_state_dtype_from_config(
cls, vllm_config: VllmConfig
) -> tuple[torch.dtype, torch.dtype]:
state_dtype = {
"float16": torch.float16,
"bfloat16": torch.bfloat16,
}.get(vllm_config.cache_config.mamba_ssm_cache_dtype, torch.float32)
return torch.float32, state_dtype
class _FakeRegistry:
@staticmethod
def resolve_model_cls(
architecture: str,
model_config: SimpleNamespace,
) -> tuple[type[_FakeModel], str]:
return _FakeModel, architecture
def _cpu_config(
cache_config: CacheConfig,
*,
model_type: str,
resolved_dtype: str,
architecture: str | None = None,
layer_types: tuple[str, ...] = ("linear_attention",),
) -> SimpleNamespace:
model_config = SimpleNamespace(
disable_cascade_attn=False,
architecture=architecture or model_type,
has_inner_state=True,
use_mla=False,
hf_text_config=SimpleNamespace(
model_type=model_type,
layer_types=layer_types,
),
registry=_FakeRegistry,
)
config = VllmConfig(cache_config=cache_config)
config.model_config = model_config
cache_config.mamba_ssm_cache_dtype = resolved_dtype
return config
@pytest.mark.parametrize(
(
"model_type",
"requested_dtype",
"resolved_dtype",
"layer_types",
"expected_dtype",
),
[
pytest.param(
"qwen3_5",
"bfloat16",
"bfloat16",
("linear_attention",),
"bfloat16",
id="gdn-explicit-bf16",
),
pytest.param(
"qwen3_5",
"float16",
"float16",
("linear_attention",),
"float16",
id="gdn-explicit-fp16",
),
pytest.param(
"qwen3_5",
"auto",
"bfloat16",
("linear_attention",),
"bfloat16",
id="gdn-model-bf16",
),
pytest.param(
"qwen3_5",
"auto",
"float16",
("linear_attention",),
"float16",
id="gdn-model-fp16",
),
pytest.param(
"nemotron_h",
"auto",
"bfloat16",
("mamba",),
"float32",
id="unsupported-model-bf16",
),
pytest.param(
"nemotron_h",
"auto",
"float16",
("mamba",),
"float32",
id="unsupported-model-fp16",
),
pytest.param(
"nemotron_h",
"bfloat16",
"bfloat16",
("mamba",),
"float32",
id="unsupported-explicit-bf16",
),
pytest.param(
"nemotron_h",
"float16",
"float16",
("mamba",),
"float32",
id="unsupported-explicit-fp16",
),
],
)
def test_cpu_accelerated_gdn_dtype_policy(
monkeypatch: pytest.MonkeyPatch,
model_type: str,
requested_dtype: str,
resolved_dtype: str,
layer_types: tuple[str, ...],
expected_dtype: str,
) -> None:
monkeypatch.setattr("torch.cpu._is_avx512_bf16_supported", lambda: True)
cache_config = CacheConfig(mamba_ssm_cache_dtype=requested_dtype)
config = _cpu_config(
cache_config,
model_type=model_type,
resolved_dtype=resolved_dtype,
layer_types=layer_types,
)
CpuPlatform.check_and_update_config(config)
assert cache_config.mamba_ssm_cache_dtype == expected_dtype