[ROCm][CI] Enable batch invariance for the ngram scheduling accuracy test (#62048)
Signed-off-by: Djordje Ramic <djoramic@amd.com>
This commit is contained in:
@@ -12,6 +12,7 @@ from typing import Any
|
|||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
from tests.conftest import HfRunner, VllmRunner
|
from tests.conftest import HfRunner, VllmRunner
|
||||||
from tests.utils import (
|
from tests.utils import (
|
||||||
large_gpu_mark,
|
large_gpu_mark,
|
||||||
@@ -168,9 +169,16 @@ def test_with_eagle3_spec_decoding(
|
|||||||
reason=("XPU matmul/attention kernels are not batch-invariant"),
|
reason=("XPU matmul/attention kernels are not batch-invariant"),
|
||||||
)
|
)
|
||||||
def test_with_ngram_gpu_spec_decoding(
|
def test_with_ngram_gpu_spec_decoding(
|
||||||
vllm_runner: type[VllmRunner], hf_runner: type[HfRunner]
|
vllm_runner: type[VllmRunner],
|
||||||
|
hf_runner: type[HfRunner],
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
):
|
):
|
||||||
"""Check ngram accuracy and acceptance across scheduling configurations."""
|
"""Check ngram accuracy and acceptance across scheduling configurations."""
|
||||||
|
if current_platform.is_rocm():
|
||||||
|
# hipBLASLt picks GEMM solutions by batch shape, so sync and async runs
|
||||||
|
# can flip near-tie greedy tokens and shift ngram acceptance.
|
||||||
|
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "1")
|
||||||
|
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
|
||||||
ngram_gpu_config = {
|
ngram_gpu_config = {
|
||||||
"method": "ngram_gpu",
|
"method": "ngram_gpu",
|
||||||
"num_speculative_tokens": 3,
|
"num_speculative_tokens": 3,
|
||||||
|
|||||||
Reference in New Issue
Block a user