[Kernel][Perf] Add fused CUDA post-conv MTP decode kernel for Qwen3.5 GDN (#51674)
Signed-off-by: Jie Fang <jief@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Fable 5
parent
d18bb7b390
commit
1be3628367
@@ -252,6 +252,11 @@ def test_splitting_ops_dynamic():
|
||||
# populated when the engine decides to use piecewise compilation.
|
||||
assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
assert config.compilation_config.splitting_ops_contain_attention()
|
||||
splitting_ops = config.compilation_config.splitting_ops
|
||||
assert splitting_ops is not None
|
||||
assert {
|
||||
"vllm::qwen_gdn_attention_core_fused_norm_packed",
|
||||
} <= set(splitting_ops)
|
||||
|
||||
# When use_inductor_graph_partition=True
|
||||
config = VllmConfig(
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Model-path tests for fused Qwen3.5 GDN MTP decode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not (current_platform.is_cuda() and current_platform.has_device_capability(80)):
|
||||
pytest.skip(
|
||||
reason="Fused GDN MTP decode requires CUDA compute capability 8.0+.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from tests.v1.attention.utils import ( # noqa: E402
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig, set_current_vllm_config # noqa: E402
|
||||
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn # noqa: E402
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( # noqa: E402
|
||||
ChunkGatedDeltaRule,
|
||||
QwenGatedDeltaNetAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import ( # noqa: E402
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( # noqa: E402
|
||||
rmsnorm_fn,
|
||||
)
|
||||
from vllm.utils.torch_utils import _encode_layer_name # noqa: E402
|
||||
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec # noqa: E402
|
||||
|
||||
NUM_SPEC = 3
|
||||
SPEC_TOKENS = NUM_SPEC + 1
|
||||
H = 1
|
||||
HV = 8
|
||||
K = 128
|
||||
V = 128
|
||||
CONV_KERNEL = 4
|
||||
CONV_DIM = 2 * H * K + HV * V
|
||||
BLOCK_SIZE = 16
|
||||
PREFIX = "model.layers.0.linear_attn"
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
class _TestGatedNorm:
|
||||
def __init__(self, weight: torch.Tensor) -> None:
|
||||
self.weight = weight
|
||||
self.bias = None
|
||||
self.eps = EPS
|
||||
self.group_size = None
|
||||
self.norm_before_gate = True
|
||||
self.activation = "silu"
|
||||
|
||||
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
|
||||
return rmsnorm_fn(
|
||||
x,
|
||||
self.weight,
|
||||
None,
|
||||
z=z,
|
||||
eps=self.eps,
|
||||
norm_before_gate=True,
|
||||
activation=self.activation,
|
||||
)
|
||||
|
||||
|
||||
def _make_vllm_config():
|
||||
config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
hf_config_override={"linear_key_head_dim": K},
|
||||
)
|
||||
config.additional_config = {"gdn_prefill_backend": "cutedsl"}
|
||||
config.cache_config.mamba_cache_mode = "none"
|
||||
config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=NUM_SPEC
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _build_layer(
|
||||
vllm_config,
|
||||
conv_state: torch.Tensor,
|
||||
ssm_state: torch.Tensor,
|
||||
a_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
):
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
enable_packed_recurrent_decode=False,
|
||||
disable_tp_for_ba_proj=False,
|
||||
tp_size=1,
|
||||
num_k_heads=H,
|
||||
num_v_heads=HV,
|
||||
head_k_dim=K,
|
||||
head_v_dim=V,
|
||||
key_dim=K,
|
||||
value_dim=HV * V,
|
||||
activation="silu",
|
||||
A_log=a_log,
|
||||
dt_bias=dt_bias,
|
||||
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
|
||||
kv_cache=(conv_state, ssm_state),
|
||||
norm=_TestGatedNorm(norm_weight),
|
||||
layer_norm_epsilon=EPS,
|
||||
gdn_decode_kernel="cuda",
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
layer.chunk_gated_delta_rule = ChunkGatedDeltaRule()
|
||||
for name in (
|
||||
"rearrange_mixed_qkv",
|
||||
"_forward_core",
|
||||
"_forward_core_decode_spec_post_conv_fused_norm",
|
||||
"_forward_core_decode_spec_fused_norm",
|
||||
"_can_use_fused_gdn_mtp_decode",
|
||||
"_rms_norm_gated_cuda",
|
||||
"_forward_core_fused_norm",
|
||||
"_forward_core_fused_norm_packed",
|
||||
"split_ba",
|
||||
):
|
||||
setattr(
|
||||
layer,
|
||||
name,
|
||||
types.MethodType(getattr(QwenGatedDeltaNetAttention, name), layer),
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_fused_forward_uses_packed_entrypoint() -> None:
|
||||
"""Fused mode keeps projected QKVZ and BA packed through the model op."""
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 3
|
||||
hidden_states = torch.empty(num_tokens, 1, dtype=torch.bfloat16, device=device)
|
||||
mixed_qkvz = torch.randn(
|
||||
num_tokens, CONV_DIM + HV * V, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
enable_fused_gdn_decode=True,
|
||||
norm=types.SimpleNamespace(
|
||||
weight=torch.empty(V, dtype=torch.bfloat16, device=device)
|
||||
),
|
||||
num_v_heads=HV,
|
||||
tp_size=1,
|
||||
head_v_dim=V,
|
||||
in_proj_qkvz=lambda _: (mixed_qkvz, None),
|
||||
in_proj_ba=lambda _: (ba, None),
|
||||
out_proj=lambda x: (x, None),
|
||||
)
|
||||
layer.forward_cuda = types.MethodType(
|
||||
QwenGatedDeltaNetAttention.forward_cuda, layer
|
||||
)
|
||||
|
||||
def packed_op(
|
||||
actual_qkvz: torch.Tensor,
|
||||
actual_ba: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
*,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
assert actual_qkvz is mixed_qkvz
|
||||
assert actual_ba is ba
|
||||
assert layer_name == _encode_layer_name(PREFIX)
|
||||
output.fill_(1)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
torch.ops.vllm,
|
||||
"qwen_gdn_attention_core_fused_norm_packed",
|
||||
side_effect=packed_op,
|
||||
) as packed_mock,
|
||||
patch.object(torch.ops.vllm, "qwen_gdn_attention_core") as triton_mock,
|
||||
):
|
||||
output = layer.forward_cuda(hidden_states)
|
||||
|
||||
packed_mock.assert_called_once()
|
||||
triton_mock.assert_not_called()
|
||||
torch.testing.assert_close(output, torch.ones_like(output))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,query_lens,draft_tokens,expected_fused_calls",
|
||||
[
|
||||
pytest.param([128], [SPEC_TOKENS], [NUM_SPEC], 1, id="pure-mtp"),
|
||||
pytest.param(
|
||||
[128, 96],
|
||||
[SPEC_TOKENS, 64],
|
||||
[NUM_SPEC, -1],
|
||||
0,
|
||||
id="mixed-mtp-falls-back",
|
||||
),
|
||||
pytest.param([96], [64], [-1], 0, id="pure-prefill"),
|
||||
pytest.param([128], [1], [-1], 0, id="pure-decode"),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_model_path_matches_reference(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
draft_tokens: list[int],
|
||||
expected_fused_calls: int,
|
||||
) -> None:
|
||||
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
|
||||
torch.manual_seed(1)
|
||||
device = torch.device("cuda")
|
||||
vllm_config = _make_vllm_config()
|
||||
builder = GDNAttentionMetadataBuilder(
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=NUM_SPEC,
|
||||
),
|
||||
layer_names=[PREFIX],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
batch = BatchSpec(seq_lens=seq_lens, query_lens=query_lens)
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, device, arange_block_indices=True
|
||||
)
|
||||
common.block_table_tensor.add_(1)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
metadata = builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common,
|
||||
num_accepted_tokens=torch.ones(
|
||||
batch.batch_size, dtype=torch.int32, device=device
|
||||
),
|
||||
num_decode_draft_tokens_cpu=torch.tensor(draft_tokens, dtype=torch.int32),
|
||||
)
|
||||
|
||||
state_indices = [
|
||||
indices
|
||||
for indices in (
|
||||
metadata.spec_state_indices_tensor,
|
||||
metadata.non_spec_state_indices_tensor,
|
||||
)
|
||||
if indices is not None and indices.numel() > 0
|
||||
]
|
||||
pool_size = max(int(indices.max().item()) for indices in state_indices) + 1
|
||||
conv_state_shape, temporal_state_shape = (
|
||||
MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||
1, H, HV, K, V, CONV_KERNEL, NUM_SPEC
|
||||
)
|
||||
)
|
||||
conv_state_seed = 0.05 * torch.randn(
|
||||
pool_size, *conv_state_shape, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
ssm_state_seed = 0.01 * torch.randn(
|
||||
pool_size, *temporal_state_shape, dtype=torch.float32, device=device
|
||||
)
|
||||
a_log = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
conv_weight = 0.1 * torch.randn(
|
||||
CONV_DIM, 1, CONV_KERNEL, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
norm_weight = torch.randn(V, dtype=torch.float32, device=device)
|
||||
num_tokens = batch.compute_num_tokens()
|
||||
mixed_qkv = 0.1 * torch.randn(
|
||||
num_tokens, CONV_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
b = 0.1 * torch.randn(num_tokens, HV, dtype=torch.bfloat16, device=device)
|
||||
a = 0.1 * torch.randn_like(b)
|
||||
output_gate = 0.1 * torch.randn(
|
||||
num_tokens, HV, V, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
mixed_qkvz = torch.cat((mixed_qkv, output_gate.flatten(1)), dim=-1)
|
||||
ba = torch.cat((b, a), dim=-1)
|
||||
context = types.SimpleNamespace(attn_metadata={PREFIX: metadata})
|
||||
|
||||
reference_layer = _build_layer(
|
||||
vllm_config,
|
||||
conv_state_seed.clone(),
|
||||
ssm_state_seed.clone(),
|
||||
a_log,
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
)
|
||||
reference_out = torch.zeros_like(output_gate)
|
||||
with patch.object(
|
||||
qwen_gdn_linear_attn, "get_forward_context", return_value=context
|
||||
):
|
||||
reference_layer._forward_core(
|
||||
mixed_qkv=mixed_qkv.clone(),
|
||||
b=b,
|
||||
a=a,
|
||||
core_attn_out=reference_out,
|
||||
)
|
||||
reference_out = reference_layer.norm(reference_out, output_gate)
|
||||
|
||||
fused_layer = _build_layer(
|
||||
vllm_config,
|
||||
conv_state_seed.clone(),
|
||||
ssm_state_seed.clone(),
|
||||
a_log,
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
)
|
||||
context.no_compile_layers = {PREFIX: fused_layer}
|
||||
fused_out = torch.zeros_like(output_gate)
|
||||
fused_op = qwen_gdn_linear_attn.ops.fused_gdn_decode_post_conv_mtp
|
||||
with (
|
||||
patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=context),
|
||||
patch.object(
|
||||
qwen_gdn_linear_attn.ops,
|
||||
"fused_gdn_decode_post_conv_mtp",
|
||||
wraps=fused_op,
|
||||
) as fused_mock,
|
||||
):
|
||||
torch.ops.vllm.qwen_gdn_attention_core_fused_norm_packed(
|
||||
mixed_qkvz.clone(),
|
||||
ba,
|
||||
fused_out,
|
||||
layer_name=_encode_layer_name(PREFIX),
|
||||
)
|
||||
|
||||
assert fused_mock.call_count == expected_fused_calls
|
||||
torch.testing.assert_close(
|
||||
fused_layer.kv_cache[0], reference_layer.kv_cache[0], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(fused_out, reference_out, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(
|
||||
fused_layer.kv_cache[1],
|
||||
reference_layer.kv_cache[1],
|
||||
atol=3e-2,
|
||||
rtol=3e-2,
|
||||
)
|
||||
@@ -1,18 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for fused_gdn_prefill_post_conv kernel.
|
||||
"""Tests for GDN post-convolution kernels.
|
||||
|
||||
Verifies that the fused kernel matches the reference:
|
||||
split → rearrange → contiguous → l2norm → gating
|
||||
The prefill tests cover preparation and the MTP tests cover recurrent state
|
||||
updates plus output normalization and gating.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.third_party.flash_linear_attention.ops import (
|
||||
fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.fused_gdn_prefill_post_conv import (
|
||||
fused_post_conv_prep,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
|
||||
rmsnorm_fn,
|
||||
)
|
||||
|
||||
|
||||
def reference_post_conv(
|
||||
@@ -207,3 +214,132 @@ def test_fused_post_conv_l0():
|
||||
)
|
||||
assert q.shape == (0, H, K)
|
||||
assert g.shape == (0, HV)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size,query_lengths,state_dtype,norm_dtype",
|
||||
[
|
||||
pytest.param(16, (4, 4), torch.bfloat16, torch.bfloat16, id="tp16-bf16"),
|
||||
pytest.param(4, (4, 4), torch.float32, torch.float32, id="tp4-fp32"),
|
||||
pytest.param(16, (4, 2, 0), torch.bfloat16, torch.float32, id="tp16-ragged"),
|
||||
pytest.param(4, (4, 2, 0), torch.float32, torch.bfloat16, id="tp4-ragged"),
|
||||
pytest.param(16, (8,), torch.float32, torch.bfloat16, id="tp16-max"),
|
||||
pytest.param(4, (8,), torch.bfloat16, torch.float32, id="tp4-max"),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_gdn_decode_post_conv_mtp_ratio8(
|
||||
tp_size: int,
|
||||
query_lengths: tuple[int, ...],
|
||||
state_dtype: torch.dtype,
|
||||
norm_dtype: torch.dtype,
|
||||
) -> None:
|
||||
if torch.cuda.get_device_capability() < (8, 0):
|
||||
pytest.skip("fused GDN decode MTP requires compute capability 8.0+")
|
||||
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
|
||||
pytest.skip("fused GDN decode MTP op is not built")
|
||||
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
H = 16 // tp_size
|
||||
HV = 128 // tp_size
|
||||
K = V = 128
|
||||
num_reqs = len(query_lengths)
|
||||
state_width = max(query_lengths)
|
||||
num_tokens = sum(query_lengths)
|
||||
num_slots = num_reqs * state_width + 1
|
||||
scale = K**-0.5
|
||||
eps = 1e-6
|
||||
|
||||
mixed_qkv = torch.randn(
|
||||
num_tokens,
|
||||
2 * H * K + HV * V,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
query, key, value = torch.split(
|
||||
mixed_qkv,
|
||||
[H * K, H * K, HV * V],
|
||||
dim=-1,
|
||||
)
|
||||
query = query.view(1, num_tokens, H, K)
|
||||
key = key.view(1, num_tokens, H, K)
|
||||
value = value.view(1, num_tokens, HV, V)
|
||||
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
|
||||
b, a = ba.chunk(2, dim=-1)
|
||||
assert not a.is_contiguous()
|
||||
assert not b.is_contiguous()
|
||||
A_log = 0.5 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
output_gate = torch.randn(num_tokens, HV, V, dtype=torch.bfloat16, device=device)
|
||||
norm_weight = torch.randn(V, dtype=norm_dtype, device=device)
|
||||
state_ref = (
|
||||
0.01 * torch.randn(num_slots, HV, V, K, dtype=torch.float32, device=device)
|
||||
).to(state_dtype)
|
||||
state_actual = state_ref.clone()
|
||||
state_indices = torch.arange(1, num_slots, dtype=torch.int32, device=device).view(
|
||||
num_reqs, state_width
|
||||
)
|
||||
cu_seqlens = torch.tensor(
|
||||
[0, *torch.tensor(query_lengths).cumsum(0).tolist()],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
num_accepted_tokens = torch.ones(num_reqs, dtype=torch.int32, device=device)
|
||||
if query_lengths[-1] == 0:
|
||||
state_indices[-1].zero_()
|
||||
|
||||
for step, accepted_tokens in enumerate((1, min(2, state_width), state_width)):
|
||||
num_accepted_tokens.fill_(accepted_tokens)
|
||||
if query_lengths[-1] == 0:
|
||||
num_accepted_tokens[-1] = 1
|
||||
raw_ref, _ = fused_sigmoid_gating_delta_rule_update(
|
||||
A_log=A_log,
|
||||
a=a,
|
||||
b=b,
|
||||
dt_bias=dt_bias,
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
initial_state=state_ref,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
scale=scale,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
expected = rmsnorm_fn(
|
||||
raw_ref.squeeze(0),
|
||||
norm_weight,
|
||||
None,
|
||||
z=output_gate,
|
||||
eps=eps,
|
||||
norm_before_gate=True,
|
||||
activation="silu",
|
||||
)
|
||||
actual = ops.fused_gdn_decode_post_conv_mtp(
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
b=b,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
state_indices=state_indices,
|
||||
cu_seqlens=cu_seqlens,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
state=state_actual,
|
||||
output_gate=output_gate,
|
||||
norm_weight=norm_weight,
|
||||
out=torch.empty_like(output_gate),
|
||||
scale=scale,
|
||||
norm_eps=eps,
|
||||
)
|
||||
|
||||
output_error = (actual.float() - expected.float()).norm()
|
||||
output_relative_l2 = output_error / expected.float().norm().clamp_min(1e-20)
|
||||
assert output_relative_l2 < 5e-4, (
|
||||
f"MTP output relative L2 mismatch at step {step}: "
|
||||
f"{output_relative_l2.item():.6g}"
|
||||
)
|
||||
|
||||
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
|
||||
|
||||
@@ -251,6 +251,21 @@ class TestEnvWithChoices:
|
||||
env_func()
|
||||
|
||||
|
||||
def test_gdn_decode_kernel_env(monkeypatch: pytest.MonkeyPatch):
|
||||
env_func = environment_variables["VLLM_GDN_DECODE_KERNEL"]
|
||||
monkeypatch.delenv("VLLM_GDN_DECODE_KERNEL", raising=False)
|
||||
assert env_func() == "cuda"
|
||||
|
||||
for value in ("cuda", "triton"):
|
||||
monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value)
|
||||
assert env_func() == value
|
||||
|
||||
for value in ("fused", "invalid"):
|
||||
monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value)
|
||||
with pytest.raises(ValueError, match="VLLM_GDN_DECODE_KERNEL"):
|
||||
env_func()
|
||||
|
||||
|
||||
class TestEnvListWithChoices:
|
||||
"""Test cases for env_list_with_choices function."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user