Support DeepSeek-V4 AMD Quark NVFP4 with emulation kernel (#47972)
Signed-off-by: Jimmy Adams <Adam_ji@outlook.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: huji <huidong.ji@amd.com> Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: huji <huidong.ji@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Andreas Karatzas
huji
Cursor
parent
0de0362ea1
commit
da788334bc
@@ -0,0 +1,15 @@
|
||||
model_name: "amd/DeepSeek-V4-Flash-NVFP4"
|
||||
accuracy_threshold: 0.92
|
||||
num_questions: 1319
|
||||
num_fewshot: 8
|
||||
startup_max_wait_seconds: 1800
|
||||
server_args: >-
|
||||
--tensor-parallel-size 8
|
||||
--max-model-len 4096
|
||||
--trust-remote-code
|
||||
--kv-cache-dtype fp8
|
||||
--tokenizer-mode deepseek_v4
|
||||
--compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
VLLM_ROCM_USE_AITER_MOE: "1"
|
||||
@@ -0,0 +1,15 @@
|
||||
model_name: "amd/DeepSeek-V4-Pro-NVFP4"
|
||||
accuracy_threshold: 0.92
|
||||
num_questions: 1319
|
||||
num_fewshot: 8
|
||||
startup_max_wait_seconds: 3600
|
||||
server_args: >-
|
||||
--tensor-parallel-size 8
|
||||
--max-model-len 4096
|
||||
--trust-remote-code
|
||||
--kv-cache-dtype fp8
|
||||
--tokenizer-mode deepseek_v4
|
||||
--compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
VLLM_ROCM_USE_AITER_MOE: "1"
|
||||
@@ -0,0 +1,8 @@
|
||||
# GFX950 model configurations for GSM8K evaluation
|
||||
# TP8 models only
|
||||
# DeepSeek-V4-Pro-NVFP4.yaml is temporarily disabled: repeated CI failures due to
|
||||
# cold NFS reads of the 850GiB checkpoint exceeding startup_max_wait_seconds
|
||||
# (see https://github.com/vllm-project/vllm/pull/47972). Re-enable once the
|
||||
# startup timeout fix has landed and is verified green.
|
||||
# DeepSeek-V4-Pro-NVFP4.yaml
|
||||
DeepSeek-V4-Flash-NVFP4.yaml
|
||||
@@ -19,8 +19,10 @@ from packaging import version
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
|
||||
QuarkConfig,
|
||||
QuarkLinearMethod,
|
||||
QuarkW8A8Fp8,
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
QuarkW8A8Int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
|
||||
@@ -80,6 +82,101 @@ def enable_pickle(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
|
||||
def test_quark_config_has_no_model_specific_fused_mappings():
|
||||
config = QuarkConfig({})
|
||||
|
||||
assert "gate_up_proj" not in config.packed_modules_mapping
|
||||
assert "fused_wqa_wkv" not in config.packed_modules_mapping
|
||||
|
||||
|
||||
def test_quark_config_preserves_existing_packed_modules_mapping():
|
||||
class CustomQuarkConfig(QuarkConfig):
|
||||
packed_modules_mapping = {"custom_proj": ["a", "b"]}
|
||||
|
||||
config = CustomQuarkConfig({})
|
||||
|
||||
assert config.packed_modules_mapping["custom_proj"] == ["a", "b"]
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_detects_per_block_config():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
|
||||
assert config._is_fp8_w8a8(weight_config, input_config)
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_rejects_per_block_static_input():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": False,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
|
||||
assert not config._is_fp8_w8a8(weight_config, input_config)
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_rejects_per_block_group_size_mismatch():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 64,
|
||||
"symmetric": True,
|
||||
}
|
||||
|
||||
assert not config._is_fp8_w8a8(weight_config, input_config)
|
||||
|
||||
|
||||
def test_quark_w8a8_fp8_per_block_requires_block_size():
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError, match="requires `block_size`"):
|
||||
QuarkW8A8Fp8PerBlock(weight_config, input_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("tp", [1])
|
||||
def test_quark_fp8_w_per_tensor_a_per_tensor(vllm_runner, kv_cache_dtype, tp):
|
||||
|
||||
Reference in New Issue
Block a user