Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
@@ -1,60 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.model_executor.warmup import kernel_warmup
|
||||
|
||||
|
||||
def test_resolve_flashinfer_autotune_file_default_layout(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
fake_jit = SimpleNamespace(
|
||||
env=SimpleNamespace(
|
||||
FLASHINFER_WORKSPACE_DIR=Path("/flashinfer-cache/0.6.11.post2/103a")
|
||||
)
|
||||
)
|
||||
fake_flashinfer = SimpleNamespace(jit=fake_jit)
|
||||
monkeypatch.setitem(sys.modules, "flashinfer", fake_flashinfer)
|
||||
monkeypatch.setitem(sys.modules, "flashinfer.jit", fake_jit)
|
||||
monkeypatch.setattr(
|
||||
kernel_warmup, "aot_compile_hash_factors", lambda _: ["env-hash", "config-hash"]
|
||||
)
|
||||
monkeypatch.setattr(kernel_warmup.envs, "VLLM_CACHE_ROOT", str(tmp_path))
|
||||
monkeypatch.setattr(kernel_warmup.envs, "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None)
|
||||
|
||||
runner = SimpleNamespace(vllm_config=SimpleNamespace())
|
||||
cache_hash = sha256(str(["env-hash", "config-hash"]).encode()).hexdigest()
|
||||
|
||||
path = kernel_warmup._resolve_flashinfer_autotune_file(runner)
|
||||
|
||||
assert path == (
|
||||
tmp_path
|
||||
/ "flashinfer_autotune_cache"
|
||||
/ "0.6.11.post2"
|
||||
/ "103a"
|
||||
/ cache_hash
|
||||
/ "autotune_configs.json"
|
||||
)
|
||||
assert path.parent.is_dir()
|
||||
|
||||
|
||||
def test_resolve_flashinfer_autotune_file_uses_override_dir(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
kernel_warmup.envs, "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", str(tmp_path)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kernel_warmup, "aot_compile_hash_factors", lambda _: ["env-hash", "config-hash"]
|
||||
)
|
||||
|
||||
runner = SimpleNamespace(vllm_config=SimpleNamespace())
|
||||
cache_hash = sha256(str(["env-hash", "config-hash"]).encode()).hexdigest()
|
||||
|
||||
path = kernel_warmup._resolve_flashinfer_autotune_file(runner)
|
||||
|
||||
assert path == tmp_path / cache_hash / "autotune_configs.json"
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Behavior checks for FlashInfer SM120 sparse MLA backend selection."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils import flashinfer as fi_utils
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseSM120Backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
def _fake_vllm_config(model_type: str) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(model_type=model_type, index_topk=2048),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_sm120_backend_uses_dedicated_backend_name() -> None:
|
||||
assert FlashInferMLASparseSM120Backend.get_name() == "FLASHINFER_MLA_SPARSE_SM120"
|
||||
assert (
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM120.get_class()
|
||||
is FlashInferMLASparseSM120Backend
|
||||
)
|
||||
|
||||
|
||||
def test_v32_glm_sm120_backend_accepts_glm_block_size(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(fi_utils, "has_flashinfer_sparse_mla_sm120", lambda: True)
|
||||
|
||||
with set_current_vllm_config(_fake_vllm_config("glm4_moe")):
|
||||
invalid_reasons = FlashInferMLASparseSM120Backend.validate_configuration(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
block_size=256,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=DeviceCapability(12, 0),
|
||||
attn_type="decoder",
|
||||
)
|
||||
|
||||
assert invalid_reasons == []
|
||||
@@ -36,7 +36,7 @@ if not current_platform.is_cuda():
|
||||
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseBackend,
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
FlashMLASparseBackend,
|
||||
@@ -174,8 +174,8 @@ def _quantize_dequantize_fp8_ds_mla(
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_cls",
|
||||
[FlashMLASparseBackend, FlashInferMLASparseBackend],
|
||||
ids=["FlashMLA", "FlashInfer"],
|
||||
[FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend],
|
||||
ids=["FlashMLA", "FlashInferTRTLLM"],
|
||||
)
|
||||
@pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys()))
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_ds_mla"])
|
||||
@@ -217,9 +217,12 @@ def test_sparse_backend_decode_correctness(
|
||||
ok, reason = flashmla.is_flashmla_sparse_supported()
|
||||
if not ok:
|
||||
pytest.skip(reason)
|
||||
elif backend_cls == FlashInferMLASparseBackend:
|
||||
if not current_platform.has_device_capability(100):
|
||||
pytest.skip("FlashInferMLASparseBackend requires SM 10.0 or higher")
|
||||
elif backend_cls == FlashInferMLASparseTRTLLMBackend:
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if device_capability is None or not backend_cls.supports_compute_capability(
|
||||
device_capability
|
||||
):
|
||||
pytest.skip("FlashInferMLASparseTRTLLMBackend requires SM 10.x capability")
|
||||
|
||||
batch_spec = SPARSE_BACKEND_BATCH_SPECS[batch_name]
|
||||
use_fp8_ds_mla_quantization = kv_cache_dtype == "fp8_ds_mla"
|
||||
|
||||
@@ -132,9 +132,9 @@ def get_available_attention_backends() -> list[str]:
|
||||
)
|
||||
|
||||
return [
|
||||
backend.name
|
||||
for backend, _ in valid_backends
|
||||
if backend not in EXCLUDED_BACKENDS
|
||||
candidate.backend.name
|
||||
for candidate in valid_backends
|
||||
if candidate.backend not in EXCLUDED_BACKENDS
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user