[CPU][GDN] Support NIXL DS convolution-state layout (#53300)

Signed-off-by: Li, Tianmu <tianmu.li@intel.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Tianmu Li
2026-09-24 14:55:20 +08:00
committed by GitHub
co-authored by OpenAI Codex Li, Jiang <jiang1.li@intel.com>
parent 153f2ba1b9
commit 67ddac3ea8
4 changed files with 383 additions and 21 deletions
+186
View File
@@ -473,6 +473,20 @@ def _conv_inputs(total_tokens: int):
return x, weight, bias
def _conv_states(
num_slots: int,
state_len: int,
dim: int = CONV_DIM,
layout: str = "SD",
) -> torch.Tensor:
if layout == "SD":
return torch.zeros(num_slots, state_len, dim, dtype=torch.bfloat16).transpose(
1, 2
)
assert layout == "DS"
return torch.zeros(num_slots, dim, state_len, dtype=torch.bfloat16)
def _sd_conv_states(
num_slots: int, state_len: int, dim: int = CONV_DIM
) -> torch.Tensor:
@@ -484,6 +498,39 @@ def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor
return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
@torch.inference_mode()
def test_causal_conv1d_update_cpu_preserves_sd_state(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""SD state and indices reach the native operator unchanged."""
num_slots = 2
state_len = CONV_KERNEL - 1
conv_state = _sd_conv_states(num_slots, state_len)
state_indices = torch.tensor([1], dtype=torch.int32)
x = torch.zeros(1, CONV_DIM, dtype=torch.bfloat16)
recorded = {}
def record_update(**kwargs):
recorded.update(kwargs)
return x
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
monkeypatch.setattr(gdn_attention.ops, "causal_conv1d_update_cpu", record_update)
out = gdn_attention._causal_conv1d_update_cpu(
x=x,
conv_states=conv_state,
weight=torch.zeros(CONV_DIM, CONV_KERNEL, dtype=torch.bfloat16),
bias=None,
silu_activation=False,
conv_state_indices=state_indices,
is_vnni=False,
)
assert out is x
assert recorded["conv_states"] is conv_state
assert recorded["conv_state_indices"] is state_indices
@torch.inference_mode()
def test_spec_aware_mixed_routing_preserves_token_order(
monkeypatch: pytest.MonkeyPatch,
@@ -1123,6 +1170,145 @@ def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
return out.transpose(0, 1).contiguous(), conv_state
def _python_native_fwd(**kwargs):
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_fn_cpu,
)
return causal_conv1d_fn_cpu(
x=kwargs["x"],
weight=kwargs["weight"],
bias=kwargs["bias"],
conv_states=kwargs["conv_states"],
query_start_loc=kwargs["query_start_loc"],
cache_indices=kwargs["cache_indices"],
has_initial_state=kwargs["has_initial_state"],
activation=kwargs["silu_activation"],
)
@torch.inference_mode()
def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
monkeypatch: pytest.MonkeyPatch,
) -> None:
dim = 8
seq_lens = [2, 3]
state_len = _STATE_LEN
state_indices = torch.tensor([2, 0], dtype=torch.int32)
query_start_loc = torch.tensor([0, 2, 5], dtype=torch.int32)
has_initial_state = torch.tensor([False, True])
x_tokens, weight, bias = _conv_inputs(sum(seq_lens))
x_tokens = x_tokens[:, :dim]
weight = weight[:dim]
bias = bias[:dim]
x = x_tokens.transpose(0, 1).contiguous()
logical_state = (
torch.arange(3 * dim * state_len, dtype=torch.float32)
.view(3, dim, state_len)
.to(torch.bfloat16)
)
state_sd = _conv_states(3, state_len, dim, "SD")
state_ds = _conv_states(3, state_len, dim, "DS")
state_sd.copy_(logical_state)
state_ds.copy_(logical_state)
recorded = []
def record_fwd(**kwargs):
recorded.append(
(
kwargs["conv_states"].shape,
kwargs["conv_states"].stride(),
kwargs["cache_indices"].clone(),
)
)
return _python_native_fwd(**kwargs)
monkeypatch.setattr(gdn_attention.ops, "causal_conv1d_fwd_cpu", record_fwd)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
out_sd = gdn_attention._causal_conv1d_fwd_cpu(
x=x,
weight=weight,
bias=bias,
conv_states=state_sd,
query_start_loc=query_start_loc,
cache_indices=state_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=False,
)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True)
out_ds = gdn_attention._causal_conv1d_fwd_cpu(
x=x,
weight=weight,
bias=bias,
conv_states=state_ds,
query_start_loc=query_start_loc,
cache_indices=state_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=False,
)
torch.testing.assert_close(out_ds, out_sd)
torch.testing.assert_close(state_ds, state_sd)
assert recorded[0][2].equal(state_indices)
assert recorded[1][0] == (2, dim, state_len)
assert recorded[1][1] == (dim * state_len, 1, dim)
assert recorded[1][2].equal(torch.arange(2, dtype=torch.int32))
@torch.inference_mode()
def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
dim = 8
seq_len = 4
state_len = seq_len + 5
state = _conv_states(4, state_len, dim, "DS")
state.copy_(
torch.arange(state.numel(), dtype=torch.float32).view_as(state).to(state.dtype)
)
original_state = state.clone()
state_indices = torch.tensor([3, 1, 0], dtype=torch.int32)
num_accepted_tokens = torch.tensor([1, 4, 2], dtype=torch.int32)
mutated_scratch = torch.arange(
state_indices.numel() * dim * state_len, dtype=torch.float32
).view(state_indices.numel(), dim, state_len)
mutated_scratch = (mutated_scratch + 200).to(state.dtype)
recorded = {}
def mutate_update(**kwargs):
recorded.update(kwargs)
kwargs["conv_states"].copy_(mutated_scratch)
return torch.zeros_like(kwargs["x"])
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True)
monkeypatch.setattr(gdn_attention.ops, "causal_conv1d_update_cpu", mutate_update)
gdn_attention._causal_conv1d_update_cpu(
x=torch.zeros(state_indices.numel(), seq_len, dim, dtype=state.dtype),
conv_states=state,
weight=torch.zeros(dim, CONV_KERNEL, dtype=state.dtype),
bias=None,
silu_activation=False,
conv_state_indices=state_indices,
is_vnni=False,
num_accepted_tokens=num_accepted_tokens,
)
expected_state = original_state.clone()
expected_state.index_copy_(0, state_indices.to(torch.long), mutated_scratch)
torch.testing.assert_close(state, expected_state)
torch.testing.assert_close(
state.index_select(0, state_indices)[:, :, _STATE_LEN:],
mutated_scratch[:, :, _STATE_LEN:],
)
torch.testing.assert_close(
recorded["conv_state_indices"],
torch.arange(state_indices.numel(), dtype=torch.int32),
)
torch.testing.assert_close(recorded["num_accepted_tokens"], num_accepted_tokens)
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
)
+74 -1
View File
@@ -1,12 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from types import SimpleNamespace
import pytest
import torch
from vllm.config import CacheConfig, VllmConfig
from vllm.config import CacheConfig, KVTransferConfig, VllmConfig
from vllm.platforms.cpu import CpuPlatform
@@ -44,6 +45,7 @@ def _cpu_config(
architecture=architecture or model_type,
has_inner_state=True,
use_mla=False,
hf_config=SimpleNamespace(model_type=model_type),
hf_text_config=SimpleNamespace(
model_type=model_type,
layer_types=layer_types,
@@ -150,3 +152,74 @@ def test_cpu_accelerated_gdn_dtype_policy(
CpuPlatform.check_and_update_config(config)
assert cache_config.mamba_ssm_cache_dtype == expected_dtype
@pytest.mark.parametrize(
(
"connector",
"extra_config",
"explicit_layout",
"avx512_bf16_supported",
"expected_layout",
),
[
pytest.param("NixlConnector", None, None, True, "DS", id="nixl"),
pytest.param("NixlPullConnector", None, None, True, "DS", id="nixl-pull"),
pytest.param("NixlPushConnector", None, None, True, "DS", id="nixl-push"),
pytest.param(
"MultiConnector",
{
"connectors": [
{
"kv_connector": "ExampleConnector",
"kv_connector_extra_config": {},
},
{
"kv_connector": "NixlConnector",
"kv_connector_extra_config": {},
},
]
},
None,
True,
"DS",
id="multi-nixl",
),
pytest.param("NixlConnector", None, "SD", True, "SD", id="explicit-override"),
pytest.param("OffloadingConnector", None, None, True, "SD", id="non-nixl"),
pytest.param(
"OffloadingConnector", None, None, False, None, id="non-nixl-no-avx"
),
],
)
def test_cpu_conv_state_layout_selection(
monkeypatch: pytest.MonkeyPatch,
connector: str,
extra_config: dict | None,
explicit_layout: str | None,
avx512_bf16_supported: bool,
expected_layout: str | None,
) -> None:
layout_env = "VLLM_SSM_CONV_STATE_LAYOUT"
monkeypatch.setattr(
"torch.cpu._is_avx512_bf16_supported",
lambda: avx512_bf16_supported,
)
kv_transfer_config = KVTransferConfig(
kv_connector=connector,
kv_connector_extra_config=extra_config or {},
kv_role="kv_both",
)
config = _cpu_config(
CacheConfig(mamba_ssm_cache_dtype="float32"),
model_type="qwen3_5",
resolved_dtype="float32",
)
config.kv_transfer_config = kv_transfer_config
monkeypatch.delenv(layout_env, raising=False)
if explicit_layout is not None:
monkeypatch.setenv(layout_env, explicit_layout)
CpuPlatform.check_and_update_config(config)
assert os.environ.get(layout_env) == expected_layout