[CPU][GDN] Support NIXL DS convolution-state layout (#53300)

Signed-off-by: Li, Tianmu <tianmu.li@intel.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Tianmu Li
2026-09-24 14:55:20 +08:00
committed by GitHub
co-authored by OpenAI Codex Li, Jiang <jiang1.li@intel.com>
parent 153f2ba1b9
commit 67ddac3ea8
4 changed files with 383 additions and 21 deletions
+74 -1
View File
@@ -1,12 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from types import SimpleNamespace
import pytest
import torch
from vllm.config import CacheConfig, VllmConfig
from vllm.config import CacheConfig, KVTransferConfig, VllmConfig
from vllm.platforms.cpu import CpuPlatform
@@ -44,6 +45,7 @@ def _cpu_config(
architecture=architecture or model_type,
has_inner_state=True,
use_mla=False,
hf_config=SimpleNamespace(model_type=model_type),
hf_text_config=SimpleNamespace(
model_type=model_type,
layer_types=layer_types,
@@ -150,3 +152,74 @@ def test_cpu_accelerated_gdn_dtype_policy(
CpuPlatform.check_and_update_config(config)
assert cache_config.mamba_ssm_cache_dtype == expected_dtype
@pytest.mark.parametrize(
(
"connector",
"extra_config",
"explicit_layout",
"avx512_bf16_supported",
"expected_layout",
),
[
pytest.param("NixlConnector", None, None, True, "DS", id="nixl"),
pytest.param("NixlPullConnector", None, None, True, "DS", id="nixl-pull"),
pytest.param("NixlPushConnector", None, None, True, "DS", id="nixl-push"),
pytest.param(
"MultiConnector",
{
"connectors": [
{
"kv_connector": "ExampleConnector",
"kv_connector_extra_config": {},
},
{
"kv_connector": "NixlConnector",
"kv_connector_extra_config": {},
},
]
},
None,
True,
"DS",
id="multi-nixl",
),
pytest.param("NixlConnector", None, "SD", True, "SD", id="explicit-override"),
pytest.param("OffloadingConnector", None, None, True, "SD", id="non-nixl"),
pytest.param(
"OffloadingConnector", None, None, False, None, id="non-nixl-no-avx"
),
],
)
def test_cpu_conv_state_layout_selection(
monkeypatch: pytest.MonkeyPatch,
connector: str,
extra_config: dict | None,
explicit_layout: str | None,
avx512_bf16_supported: bool,
expected_layout: str | None,
) -> None:
layout_env = "VLLM_SSM_CONV_STATE_LAYOUT"
monkeypatch.setattr(
"torch.cpu._is_avx512_bf16_supported",
lambda: avx512_bf16_supported,
)
kv_transfer_config = KVTransferConfig(
kv_connector=connector,
kv_connector_extra_config=extra_config or {},
kv_role="kv_both",
)
config = _cpu_config(
CacheConfig(mamba_ssm_cache_dtype="float32"),
model_type="qwen3_5",
resolved_dtype="float32",
)
config.kv_transfer_config = kv_transfer_config
monkeypatch.delenv(layout_env, raising=False)
if explicit_layout is not None:
monkeypatch.setenv(layout_env, explicit_layout)
CpuPlatform.check_and_update_config(config)
assert os.environ.get(layout_env) == expected_layout