[CI] Exercise FA3 FP8 attention on SM90 (#49231)
Signed-off-by: Simon Mo <simon@inferact.ai>
This commit is contained in:
@@ -93,9 +93,6 @@ def test_online_quantization(
|
||||
use_rocm_aiter: bool,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
|
||||
pytest.skip("FA3 currently rejects FP8 KV cache output dtype on SM90")
|
||||
|
||||
if use_rocm_aiter:
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
@@ -105,9 +102,15 @@ def test_online_quantization(
|
||||
if force_marlin:
|
||||
monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")
|
||||
|
||||
model_dtype = "auto"
|
||||
if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
|
||||
# FA3 requires BF16 output when the query input is FP8.
|
||||
model_dtype = "bfloat16"
|
||||
|
||||
with vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
quantization="fp8",
|
||||
dtype=model_dtype,
|
||||
enforce_eager=True,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
) as llm:
|
||||
|
||||
Reference in New Issue
Block a user