[Test][LoRA] Speed up the LoRA test job (#52331)

Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
This commit is contained in:
stefankoncarevic
2026-08-14 12:08:41 -05:00
committed by GitHub
parent e078a2238a
commit 83ded8d839
4 changed files with 104 additions and 126 deletions
+4
View File
@@ -22,6 +22,10 @@ from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.torch_utils import set_random_seed
# The tensor-parallel cases build their distributed environment inside processes
# spawned by torch.multiprocessing.spawn, so this process never has one to tear down.
pytestmark = pytest.mark.skip_global_cleanup
@pytest.fixture(autouse=True)
def reset_device(reset_default_device):
+12 -7
View File
@@ -56,14 +56,19 @@ TOLERANCES = {
torch.bfloat16: (3e-2, 2e-2),
}
pytestmark = pytest.mark.skipif(
not (
current_platform.is_cuda_alike()
or current_platform.is_cpu()
or current_platform.is_xpu()
pytestmark = [
pytest.mark.skipif(
not (
current_platform.is_cuda_alike()
or current_platform.is_cpu()
or current_platform.is_xpu()
),
reason="Backend not supported",
),
reason="Backend not supported",
)
# Tests here either take dist_init, which tears the distributed environment
# down itself, or never build one, so the global cleanup only repeats it.
pytest.mark.skip_global_cleanup,
]
DEVICE_TYPE = current_platform.device_type
DEVICES = (
+37 -27
View File
@@ -15,6 +15,14 @@ from .utils import PunicaTensors, assert_close, generate_data_for_nslices
DEVICE_TYPE = current_platform.device_type
# On XPU, oneDNN/oneMKL can return wrong results for these reference matmuls
# after many Triton kernel launches, so the reference stays on CPU there.
_REF_ON_CPU = current_platform.is_xpu()
def _to_ref_device(tensor: torch.Tensor) -> torch.Tensor:
return tensor.cpu() if _REF_ON_CPU else tensor
@pytest.fixture(autouse=True)
def reset_device(reset_default_device):
@@ -33,10 +41,10 @@ def dynamo_reset():
yield
def _cpu_bgmv_shrink(
def _bgmv_shrink(
inputs, lora_weight, output, seq_len_tensor, lora_indices, scaling=1.0
):
"""Memory-efficient shrink reference: per-LoRA matmul loop on CPU.
"""Memory-efficient shrink reference: per-LoRA matmul loop.
output[mask] = scaling * inputs[mask] @ weight.T"""
exploded = torch.repeat_interleave(lora_indices, seq_len_tensor)
for lid in exploded.unique():
@@ -48,7 +56,7 @@ def _cpu_bgmv_shrink(
output[mask] = scaling * (inp @ w.T)
def _cpu_bgmv_expand(
def _bgmv_expand(
inputs,
lora_weight,
output,
@@ -57,7 +65,7 @@ def _cpu_bgmv_expand(
offset=0,
add_inputs=False,
):
"""Memory-efficient expand reference: per-LoRA matmul loop on CPU.
"""Memory-efficient expand reference: per-LoRA matmul loop.
output[mask, offset:offset+n] (+)= inputs[mask] @ weight.T"""
exploded = torch.repeat_interleave(lora_indices, seq_len_tensor)
for lid in exploded.unique():
@@ -88,21 +96,22 @@ def sgmv_shrink_for_nslices(
num_tokens: int,
scaling: float,
):
"""CPU reference for sgmv_shrink using per-LoRA matmul loop."""
inp_cpu = inputs_tensor.cpu()
seq_cpu = seq_len_tensor.cpu()
idx_cpu = prompt_lora_mapping.cpu()
out_cpu = out_tensor.cpu()
"""Reference for sgmv_shrink using per-LoRA matmul loop."""
inputs = _to_ref_device(inputs_tensor)
seq_len = _to_ref_device(seq_len_tensor)
mapping = _to_ref_device(prompt_lora_mapping)
out = _to_ref_device(out_tensor)
for index in range(nslices):
_cpu_bgmv_shrink(
inp_cpu,
lora_weights_lst[index].cpu(),
out_cpu[index],
seq_cpu,
idx_cpu,
_bgmv_shrink(
inputs,
_to_ref_device(lora_weights_lst[index]),
out[index],
seq_len,
mapping,
scaling=scaling,
)
out_tensor.copy_(out_cpu)
if _REF_ON_CPU:
out_tensor.copy_(out)
def sgmv_expand_for_nslices(
@@ -119,21 +128,22 @@ def sgmv_expand_for_nslices(
num_tokens: int,
add_inputs: bool,
) -> None:
"""CPU reference for sgmv_expand using per-LoRA matmul loop."""
seq_cpu = seq_len_tensor.cpu()
idx_cpu = prompt_lora_mapping.cpu()
out_cpu = out_tensor.cpu()
"""Reference for sgmv_expand using per-LoRA matmul loop."""
seq_len = _to_ref_device(seq_len_tensor)
mapping = _to_ref_device(prompt_lora_mapping)
out = _to_ref_device(out_tensor)
for index in range(nslices):
_cpu_bgmv_expand(
inputs_tensor[index].cpu(),
lora_weights_lst[index].cpu(),
out_cpu,
seq_cpu,
idx_cpu,
_bgmv_expand(
_to_ref_device(inputs_tensor[index]),
_to_ref_device(lora_weights_lst[index]),
out,
seq_len,
mapping,
offset=hidden_size * index,
add_inputs=add_inputs,
)
out_tensor.copy_(out_cpu)
if _REF_ON_CPU:
out_tensor.copy_(out)
_dict_lock = Lock()
+51 -92
View File
@@ -17,6 +17,7 @@ from threading import Lock
import pytest
import torch
import torch.nn.functional as F
import vllm.lora.ops.torch_ops as torch_ops
import vllm.lora.ops.triton_ops as triton_ops
@@ -31,6 +32,8 @@ from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
pytestmark = pytest.mark.skip_global_cleanup
DEVICE_TYPE = current_platform.device_type
DEVICES = [f"{DEVICE_TYPE}:{0}"]
SEED = [0]
@@ -206,43 +209,32 @@ def quantize_to_fp8_blockwise(
if tensor.ndim == 2:
M, K = tensor.shape
n_blocks_k = math.ceil(K / group_k)
scale = torch.zeros(M, n_blocks_k, dtype=torch.float32, device=tensor.device)
fp8_tensor = torch.zeros_like(tensor, dtype=FP8_DTYPE)
for m in range(M):
for bk in range(n_blocks_k):
k_start = bk * group_k
k_end = min(k_start + group_k, K)
block = tensor[m, k_start:k_end].float()
amax = block.abs().max().clamp(min=1e-12)
s = (amax / FP8_MAX).to(torch.float32)
scale[m, bk] = s
fp8_tensor[m, k_start:k_end] = (
(block / s).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
)
return fp8_tensor, scale
# Zero padding leaves the per-block amax unchanged, so the padded
# columns are dropped again after quantizing.
padded = F.pad(tensor.float(), (0, n_blocks_k * group_k - K))
blocks = padded.view(M, n_blocks_k, group_k)
scale = blocks.abs().amax(dim=-1).clamp(min=1e-12) / FP8_MAX
fp8_tensor = (
(blocks / scale.unsqueeze(-1)).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
)
return fp8_tensor.view(M, -1)[:, :K].contiguous(), scale
elif tensor.ndim == 3:
L, N, K = tensor.shape
n_blocks_n = math.ceil(N / group_n)
n_blocks_k = math.ceil(K / group_k)
scale = torch.zeros(
L, n_blocks_n, n_blocks_k, dtype=torch.float32, device=tensor.device
padded = F.pad(
tensor.float(),
(0, n_blocks_k * group_k - K, 0, n_blocks_n * group_n - N),
)
fp8_tensor = torch.zeros_like(tensor, dtype=FP8_DTYPE)
for li in range(L):
for bn in range(n_blocks_n):
for bk in range(n_blocks_k):
n_start = bn * group_n
n_end = min(n_start + group_n, N)
k_start = bk * group_k
k_end = min(k_start + group_k, K)
block = tensor[li, n_start:n_end, k_start:k_end].float()
amax = block.abs().max().clamp(min=1e-12)
s = (amax / FP8_MAX).to(torch.float32)
scale[li, bn, bk] = s
fp8_tensor[li, n_start:n_end, k_start:k_end] = (
(block / s).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
)
return fp8_tensor, scale
blocks = padded.view(L, n_blocks_n, group_n, n_blocks_k, group_k)
scale = blocks.abs().amax(dim=(2, 4)).clamp(min=1e-12) / FP8_MAX
fp8_tensor = (
(blocks / scale[:, :, None, :, None])
.clamp(FP8_MIN, FP8_MAX)
.to(FP8_DTYPE)
.view(L, n_blocks_n * group_n, n_blocks_k * group_k)
)
return fp8_tensor[:, :N, :K].contiguous(), scale
else:
raise ValueError(f"Unsupported tensor ndim: {tensor.ndim}")
@@ -301,33 +293,23 @@ def dequantize_fp8_blockwise(
"""Dequantize FP8 tensor with block-wise scale back to output_dtype."""
if fp8_tensor.ndim == 2:
M, K = fp8_tensor.shape
out = torch.zeros(M, K, dtype=output_dtype, device=fp8_tensor.device)
n_blocks_k = math.ceil(K / group_k)
for m in range(M):
for bk in range(n_blocks_k):
k_start = bk * group_k
k_end = min(k_start + group_k, K)
out[m, k_start:k_end] = (
fp8_tensor[m, k_start:k_end].float() * scale[m, bk].float()
).to(output_dtype)
return out
padded = F.pad(fp8_tensor.float(), (0, n_blocks_k * group_k - K))
blocks = padded.view(M, n_blocks_k, group_k)
out = (blocks * scale.float().unsqueeze(-1)).to(output_dtype)
return out.view(M, -1)[:, :K].contiguous()
elif fp8_tensor.ndim == 3:
L, N, K = fp8_tensor.shape
out = torch.zeros(L, N, K, dtype=output_dtype, device=fp8_tensor.device)
n_blocks_n = math.ceil(N / group_n)
n_blocks_k = math.ceil(K / group_k)
for l_idx in range(L):
for bn in range(n_blocks_n):
for bk in range(n_blocks_k):
n_start = bn * group_n
n_end = min(n_start + group_n, N)
k_start = bk * group_k
k_end = min(k_start + group_k, K)
out[l_idx, n_start:n_end, k_start:k_end] = (
fp8_tensor[l_idx, n_start:n_end, k_start:k_end].float()
* scale[l_idx, bn, bk].float()
).to(output_dtype)
return out
padded = F.pad(
fp8_tensor.float(),
(0, n_blocks_k * group_k - K, 0, n_blocks_n * group_n - N),
)
blocks = padded.view(L, n_blocks_n, group_n, n_blocks_k, group_k)
out = (blocks * scale.float()[:, :, None, :, None]).to(output_dtype)
out = out.view(L, n_blocks_n * group_n, n_blocks_k * group_k)
return out[:, :N, :K].contiguous()
else:
raise ValueError(f"Unsupported tensor ndim: {fp8_tensor.ndim}")
@@ -519,45 +501,22 @@ def generate_fp8_expand_data(
# shared across slices. Compute shared scale across slices, then quantize.
# First compute per-token-per-block scale across all slices
n_blocks_k = math.ceil(rank / group_k)
a_scale = torch.zeros(
total_tokens, n_blocks_k, dtype=torch.float32, device=device
padded = F.pad(inputs_bf16.float(), (0, n_blocks_k * group_k - rank))
blocks = padded.view(nslices, total_tokens, n_blocks_k, group_k)
# Take the block amax across all slices so every slice shares the scale.
a_scale = blocks.abs().amax(dim=-1).clamp(min=1e-12).amax(dim=0) / FP8_MAX
fp8_blocks = (
(blocks / a_scale[None, :, :, None]).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
)
inputs_fp8 = fp8_blocks.view(nslices, total_tokens, -1)[
:, :, :rank
].contiguous()
inputs_dequant = (
(fp8_blocks.float() * a_scale[None, :, :, None])
.to(dtype)
.view(nslices, total_tokens, -1)[:, :, :rank]
.contiguous()
)
for m in range(total_tokens):
for bk in range(n_blocks_k):
k_start = bk * group_k
k_end = min(k_start + group_k, rank)
# Max across all slices for this token and block
block_amax = torch.tensor(0.0, device=device)
for s in range(nslices):
block = inputs_bf16[s, m, k_start:k_end].float()
block_amax = torch.max(
block_amax, block.abs().max().clamp(min=1e-12)
)
a_scale[m, bk] = (block_amax / FP8_MAX).to(torch.float32)
# Quantize all slices with the shared scale
inputs_fp8_list = []
inputs_dequant_list = []
for s in range(nslices):
slice_2d = inputs_bf16[s] # (total_tokens, rank)
fp8_slice = torch.zeros_like(slice_2d, dtype=FP8_DTYPE)
dequant_slice = torch.zeros_like(slice_2d)
for m in range(total_tokens):
for bk in range(n_blocks_k):
k_start = bk * group_k
k_end = min(k_start + group_k, rank)
block = slice_2d[m, k_start:k_end].float()
s_val = a_scale[m, bk]
fp8_slice[m, k_start:k_end] = (
(block / s_val).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
)
dequant_slice[m, k_start:k_end] = (
fp8_slice[m, k_start:k_end].float() * s_val.float()
).to(dtype)
inputs_fp8_list.append(fp8_slice)
inputs_dequant_list.append(dequant_slice)
inputs_fp8 = torch.stack(inputs_fp8_list, dim=0)
inputs_dequant = torch.stack(inputs_dequant_list, dim=0)
elif quant_mode == "per_tensor":
# Per-tensor: kernel loads a single scalar from a_scale_ptr
inputs_fp8_2d, a_scale = quantize_to_fp8_per_tensor(inputs_2d_all)