[CPU][GDN] Support NIXL DS convolution-state layout (#53300)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
OpenAI Codex
Li, Jiang <jiang1.li@intel.com>
parent
153f2ba1b9
commit
67ddac3ea8
@@ -1,12 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.config import CacheConfig, KVTransferConfig, VllmConfig
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
|
||||
|
||||
@@ -44,6 +45,7 @@ def _cpu_config(
|
||||
architecture=architecture or model_type,
|
||||
has_inner_state=True,
|
||||
use_mla=False,
|
||||
hf_config=SimpleNamespace(model_type=model_type),
|
||||
hf_text_config=SimpleNamespace(
|
||||
model_type=model_type,
|
||||
layer_types=layer_types,
|
||||
@@ -150,3 +152,74 @@ def test_cpu_accelerated_gdn_dtype_policy(
|
||||
|
||||
CpuPlatform.check_and_update_config(config)
|
||||
assert cache_config.mamba_ssm_cache_dtype == expected_dtype
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"connector",
|
||||
"extra_config",
|
||||
"explicit_layout",
|
||||
"avx512_bf16_supported",
|
||||
"expected_layout",
|
||||
),
|
||||
[
|
||||
pytest.param("NixlConnector", None, None, True, "DS", id="nixl"),
|
||||
pytest.param("NixlPullConnector", None, None, True, "DS", id="nixl-pull"),
|
||||
pytest.param("NixlPushConnector", None, None, True, "DS", id="nixl-push"),
|
||||
pytest.param(
|
||||
"MultiConnector",
|
||||
{
|
||||
"connectors": [
|
||||
{
|
||||
"kv_connector": "ExampleConnector",
|
||||
"kv_connector_extra_config": {},
|
||||
},
|
||||
{
|
||||
"kv_connector": "NixlConnector",
|
||||
"kv_connector_extra_config": {},
|
||||
},
|
||||
]
|
||||
},
|
||||
None,
|
||||
True,
|
||||
"DS",
|
||||
id="multi-nixl",
|
||||
),
|
||||
pytest.param("NixlConnector", None, "SD", True, "SD", id="explicit-override"),
|
||||
pytest.param("OffloadingConnector", None, None, True, "SD", id="non-nixl"),
|
||||
pytest.param(
|
||||
"OffloadingConnector", None, None, False, None, id="non-nixl-no-avx"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_cpu_conv_state_layout_selection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
connector: str,
|
||||
extra_config: dict | None,
|
||||
explicit_layout: str | None,
|
||||
avx512_bf16_supported: bool,
|
||||
expected_layout: str | None,
|
||||
) -> None:
|
||||
layout_env = "VLLM_SSM_CONV_STATE_LAYOUT"
|
||||
monkeypatch.setattr(
|
||||
"torch.cpu._is_avx512_bf16_supported",
|
||||
lambda: avx512_bf16_supported,
|
||||
)
|
||||
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector=connector,
|
||||
kv_connector_extra_config=extra_config or {},
|
||||
kv_role="kv_both",
|
||||
)
|
||||
config = _cpu_config(
|
||||
CacheConfig(mamba_ssm_cache_dtype="float32"),
|
||||
model_type="qwen3_5",
|
||||
resolved_dtype="float32",
|
||||
)
|
||||
config.kv_transfer_config = kv_transfer_config
|
||||
monkeypatch.delenv(layout_env, raising=False)
|
||||
if explicit_layout is not None:
|
||||
monkeypatch.setenv(layout_env, explicit_layout)
|
||||
|
||||
CpuPlatform.check_and_update_config(config)
|
||||
assert os.environ.get(layout_env) == expected_layout
|
||||
|
||||
Reference in New Issue
Block a user