[CPU] Support FP16/BF16 persisted GDN state on AMX (#52191)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
2a61f060d3
commit
c6c33f2b1f
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
|
||||
|
||||
class _FakeModel:
|
||||
@classmethod
|
||||
def get_mamba_state_dtype_from_config(
|
||||
cls, vllm_config: VllmConfig
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
state_dtype = {
|
||||
"float16": torch.float16,
|
||||
"bfloat16": torch.bfloat16,
|
||||
}.get(vllm_config.cache_config.mamba_ssm_cache_dtype, torch.float32)
|
||||
return torch.float32, state_dtype
|
||||
|
||||
|
||||
class _FakeRegistry:
|
||||
@staticmethod
|
||||
def resolve_model_cls(
|
||||
architecture: str,
|
||||
model_config: SimpleNamespace,
|
||||
) -> tuple[type[_FakeModel], str]:
|
||||
return _FakeModel, architecture
|
||||
|
||||
|
||||
def _cpu_config(
|
||||
cache_config: CacheConfig,
|
||||
*,
|
||||
model_type: str,
|
||||
resolved_dtype: str,
|
||||
architecture: str | None = None,
|
||||
layer_types: tuple[str, ...] = ("linear_attention",),
|
||||
) -> SimpleNamespace:
|
||||
model_config = SimpleNamespace(
|
||||
disable_cascade_attn=False,
|
||||
architecture=architecture or model_type,
|
||||
has_inner_state=True,
|
||||
use_mla=False,
|
||||
hf_text_config=SimpleNamespace(
|
||||
model_type=model_type,
|
||||
layer_types=layer_types,
|
||||
),
|
||||
registry=_FakeRegistry,
|
||||
)
|
||||
config = VllmConfig(cache_config=cache_config)
|
||||
config.model_config = model_config
|
||||
cache_config.mamba_ssm_cache_dtype = resolved_dtype
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"model_type",
|
||||
"requested_dtype",
|
||||
"resolved_dtype",
|
||||
"layer_types",
|
||||
"expected_dtype",
|
||||
),
|
||||
[
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"bfloat16",
|
||||
"bfloat16",
|
||||
("linear_attention",),
|
||||
"bfloat16",
|
||||
id="gdn-explicit-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"float16",
|
||||
"float16",
|
||||
("linear_attention",),
|
||||
"float16",
|
||||
id="gdn-explicit-fp16",
|
||||
),
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"auto",
|
||||
"bfloat16",
|
||||
("linear_attention",),
|
||||
"bfloat16",
|
||||
id="gdn-model-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"auto",
|
||||
"float16",
|
||||
("linear_attention",),
|
||||
"float16",
|
||||
id="gdn-model-fp16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"auto",
|
||||
"bfloat16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-model-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"auto",
|
||||
"float16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-model-fp16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"bfloat16",
|
||||
"bfloat16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-explicit-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"float16",
|
||||
"float16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-explicit-fp16",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_cpu_accelerated_gdn_dtype_policy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_type: str,
|
||||
requested_dtype: str,
|
||||
resolved_dtype: str,
|
||||
layer_types: tuple[str, ...],
|
||||
expected_dtype: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr("torch.cpu._is_avx512_bf16_supported", lambda: True)
|
||||
cache_config = CacheConfig(mamba_ssm_cache_dtype=requested_dtype)
|
||||
config = _cpu_config(
|
||||
cache_config,
|
||||
model_type=model_type,
|
||||
resolved_dtype=resolved_dtype,
|
||||
layer_types=layer_types,
|
||||
)
|
||||
|
||||
CpuPlatform.check_and_update_config(config)
|
||||
assert cache_config.mamba_ssm_cache_dtype == expected_dtype
|
||||
Reference in New Issue
Block a user