[Test][LoRA] Speed up the LoRA test job (#52331)
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
This commit is contained in:
@@ -22,6 +22,10 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
# The tensor-parallel cases build their distributed environment inside processes
|
||||
# spawned by torch.multiprocessing.spawn, so this process never has one to tear down.
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_device(reset_default_device):
|
||||
|
||||
@@ -56,14 +56,19 @@ TOLERANCES = {
|
||||
torch.bfloat16: (3e-2, 2e-2),
|
||||
}
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda_alike()
|
||||
or current_platform.is_cpu()
|
||||
or current_platform.is_xpu()
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda_alike()
|
||||
or current_platform.is_cpu()
|
||||
or current_platform.is_xpu()
|
||||
),
|
||||
reason="Backend not supported",
|
||||
),
|
||||
reason="Backend not supported",
|
||||
)
|
||||
# Tests here either take dist_init, which tears the distributed environment
|
||||
# down itself, or never build one, so the global cleanup only repeats it.
|
||||
pytest.mark.skip_global_cleanup,
|
||||
]
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
DEVICES = (
|
||||
|
||||
@@ -15,6 +15,14 @@ from .utils import PunicaTensors, assert_close, generate_data_for_nslices
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
# On XPU, oneDNN/oneMKL can return wrong results for these reference matmuls
|
||||
# after many Triton kernel launches, so the reference stays on CPU there.
|
||||
_REF_ON_CPU = current_platform.is_xpu()
|
||||
|
||||
|
||||
def _to_ref_device(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return tensor.cpu() if _REF_ON_CPU else tensor
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_device(reset_default_device):
|
||||
@@ -33,10 +41,10 @@ def dynamo_reset():
|
||||
yield
|
||||
|
||||
|
||||
def _cpu_bgmv_shrink(
|
||||
def _bgmv_shrink(
|
||||
inputs, lora_weight, output, seq_len_tensor, lora_indices, scaling=1.0
|
||||
):
|
||||
"""Memory-efficient shrink reference: per-LoRA matmul loop on CPU.
|
||||
"""Memory-efficient shrink reference: per-LoRA matmul loop.
|
||||
output[mask] = scaling * inputs[mask] @ weight.T"""
|
||||
exploded = torch.repeat_interleave(lora_indices, seq_len_tensor)
|
||||
for lid in exploded.unique():
|
||||
@@ -48,7 +56,7 @@ def _cpu_bgmv_shrink(
|
||||
output[mask] = scaling * (inp @ w.T)
|
||||
|
||||
|
||||
def _cpu_bgmv_expand(
|
||||
def _bgmv_expand(
|
||||
inputs,
|
||||
lora_weight,
|
||||
output,
|
||||
@@ -57,7 +65,7 @@ def _cpu_bgmv_expand(
|
||||
offset=0,
|
||||
add_inputs=False,
|
||||
):
|
||||
"""Memory-efficient expand reference: per-LoRA matmul loop on CPU.
|
||||
"""Memory-efficient expand reference: per-LoRA matmul loop.
|
||||
output[mask, offset:offset+n] (+)= inputs[mask] @ weight.T"""
|
||||
exploded = torch.repeat_interleave(lora_indices, seq_len_tensor)
|
||||
for lid in exploded.unique():
|
||||
@@ -88,21 +96,22 @@ def sgmv_shrink_for_nslices(
|
||||
num_tokens: int,
|
||||
scaling: float,
|
||||
):
|
||||
"""CPU reference for sgmv_shrink using per-LoRA matmul loop."""
|
||||
inp_cpu = inputs_tensor.cpu()
|
||||
seq_cpu = seq_len_tensor.cpu()
|
||||
idx_cpu = prompt_lora_mapping.cpu()
|
||||
out_cpu = out_tensor.cpu()
|
||||
"""Reference for sgmv_shrink using per-LoRA matmul loop."""
|
||||
inputs = _to_ref_device(inputs_tensor)
|
||||
seq_len = _to_ref_device(seq_len_tensor)
|
||||
mapping = _to_ref_device(prompt_lora_mapping)
|
||||
out = _to_ref_device(out_tensor)
|
||||
for index in range(nslices):
|
||||
_cpu_bgmv_shrink(
|
||||
inp_cpu,
|
||||
lora_weights_lst[index].cpu(),
|
||||
out_cpu[index],
|
||||
seq_cpu,
|
||||
idx_cpu,
|
||||
_bgmv_shrink(
|
||||
inputs,
|
||||
_to_ref_device(lora_weights_lst[index]),
|
||||
out[index],
|
||||
seq_len,
|
||||
mapping,
|
||||
scaling=scaling,
|
||||
)
|
||||
out_tensor.copy_(out_cpu)
|
||||
if _REF_ON_CPU:
|
||||
out_tensor.copy_(out)
|
||||
|
||||
|
||||
def sgmv_expand_for_nslices(
|
||||
@@ -119,21 +128,22 @@ def sgmv_expand_for_nslices(
|
||||
num_tokens: int,
|
||||
add_inputs: bool,
|
||||
) -> None:
|
||||
"""CPU reference for sgmv_expand using per-LoRA matmul loop."""
|
||||
seq_cpu = seq_len_tensor.cpu()
|
||||
idx_cpu = prompt_lora_mapping.cpu()
|
||||
out_cpu = out_tensor.cpu()
|
||||
"""Reference for sgmv_expand using per-LoRA matmul loop."""
|
||||
seq_len = _to_ref_device(seq_len_tensor)
|
||||
mapping = _to_ref_device(prompt_lora_mapping)
|
||||
out = _to_ref_device(out_tensor)
|
||||
for index in range(nslices):
|
||||
_cpu_bgmv_expand(
|
||||
inputs_tensor[index].cpu(),
|
||||
lora_weights_lst[index].cpu(),
|
||||
out_cpu,
|
||||
seq_cpu,
|
||||
idx_cpu,
|
||||
_bgmv_expand(
|
||||
_to_ref_device(inputs_tensor[index]),
|
||||
_to_ref_device(lora_weights_lst[index]),
|
||||
out,
|
||||
seq_len,
|
||||
mapping,
|
||||
offset=hidden_size * index,
|
||||
add_inputs=add_inputs,
|
||||
)
|
||||
out_tensor.copy_(out_cpu)
|
||||
if _REF_ON_CPU:
|
||||
out_tensor.copy_(out)
|
||||
|
||||
|
||||
_dict_lock = Lock()
|
||||
|
||||
@@ -17,6 +17,7 @@ from threading import Lock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm.lora.ops.torch_ops as torch_ops
|
||||
import vllm.lora.ops.triton_ops as triton_ops
|
||||
@@ -31,6 +32,8 @@ from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
DEVICES = [f"{DEVICE_TYPE}:{0}"]
|
||||
SEED = [0]
|
||||
@@ -206,43 +209,32 @@ def quantize_to_fp8_blockwise(
|
||||
if tensor.ndim == 2:
|
||||
M, K = tensor.shape
|
||||
n_blocks_k = math.ceil(K / group_k)
|
||||
scale = torch.zeros(M, n_blocks_k, dtype=torch.float32, device=tensor.device)
|
||||
fp8_tensor = torch.zeros_like(tensor, dtype=FP8_DTYPE)
|
||||
for m in range(M):
|
||||
for bk in range(n_blocks_k):
|
||||
k_start = bk * group_k
|
||||
k_end = min(k_start + group_k, K)
|
||||
block = tensor[m, k_start:k_end].float()
|
||||
amax = block.abs().max().clamp(min=1e-12)
|
||||
s = (amax / FP8_MAX).to(torch.float32)
|
||||
scale[m, bk] = s
|
||||
fp8_tensor[m, k_start:k_end] = (
|
||||
(block / s).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
|
||||
)
|
||||
return fp8_tensor, scale
|
||||
# Zero padding leaves the per-block amax unchanged, so the padded
|
||||
# columns are dropped again after quantizing.
|
||||
padded = F.pad(tensor.float(), (0, n_blocks_k * group_k - K))
|
||||
blocks = padded.view(M, n_blocks_k, group_k)
|
||||
scale = blocks.abs().amax(dim=-1).clamp(min=1e-12) / FP8_MAX
|
||||
fp8_tensor = (
|
||||
(blocks / scale.unsqueeze(-1)).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
|
||||
)
|
||||
return fp8_tensor.view(M, -1)[:, :K].contiguous(), scale
|
||||
elif tensor.ndim == 3:
|
||||
L, N, K = tensor.shape
|
||||
n_blocks_n = math.ceil(N / group_n)
|
||||
n_blocks_k = math.ceil(K / group_k)
|
||||
scale = torch.zeros(
|
||||
L, n_blocks_n, n_blocks_k, dtype=torch.float32, device=tensor.device
|
||||
padded = F.pad(
|
||||
tensor.float(),
|
||||
(0, n_blocks_k * group_k - K, 0, n_blocks_n * group_n - N),
|
||||
)
|
||||
fp8_tensor = torch.zeros_like(tensor, dtype=FP8_DTYPE)
|
||||
for li in range(L):
|
||||
for bn in range(n_blocks_n):
|
||||
for bk in range(n_blocks_k):
|
||||
n_start = bn * group_n
|
||||
n_end = min(n_start + group_n, N)
|
||||
k_start = bk * group_k
|
||||
k_end = min(k_start + group_k, K)
|
||||
block = tensor[li, n_start:n_end, k_start:k_end].float()
|
||||
amax = block.abs().max().clamp(min=1e-12)
|
||||
s = (amax / FP8_MAX).to(torch.float32)
|
||||
scale[li, bn, bk] = s
|
||||
fp8_tensor[li, n_start:n_end, k_start:k_end] = (
|
||||
(block / s).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
|
||||
)
|
||||
return fp8_tensor, scale
|
||||
blocks = padded.view(L, n_blocks_n, group_n, n_blocks_k, group_k)
|
||||
scale = blocks.abs().amax(dim=(2, 4)).clamp(min=1e-12) / FP8_MAX
|
||||
fp8_tensor = (
|
||||
(blocks / scale[:, :, None, :, None])
|
||||
.clamp(FP8_MIN, FP8_MAX)
|
||||
.to(FP8_DTYPE)
|
||||
.view(L, n_blocks_n * group_n, n_blocks_k * group_k)
|
||||
)
|
||||
return fp8_tensor[:, :N, :K].contiguous(), scale
|
||||
else:
|
||||
raise ValueError(f"Unsupported tensor ndim: {tensor.ndim}")
|
||||
|
||||
@@ -301,33 +293,23 @@ def dequantize_fp8_blockwise(
|
||||
"""Dequantize FP8 tensor with block-wise scale back to output_dtype."""
|
||||
if fp8_tensor.ndim == 2:
|
||||
M, K = fp8_tensor.shape
|
||||
out = torch.zeros(M, K, dtype=output_dtype, device=fp8_tensor.device)
|
||||
n_blocks_k = math.ceil(K / group_k)
|
||||
for m in range(M):
|
||||
for bk in range(n_blocks_k):
|
||||
k_start = bk * group_k
|
||||
k_end = min(k_start + group_k, K)
|
||||
out[m, k_start:k_end] = (
|
||||
fp8_tensor[m, k_start:k_end].float() * scale[m, bk].float()
|
||||
).to(output_dtype)
|
||||
return out
|
||||
padded = F.pad(fp8_tensor.float(), (0, n_blocks_k * group_k - K))
|
||||
blocks = padded.view(M, n_blocks_k, group_k)
|
||||
out = (blocks * scale.float().unsqueeze(-1)).to(output_dtype)
|
||||
return out.view(M, -1)[:, :K].contiguous()
|
||||
elif fp8_tensor.ndim == 3:
|
||||
L, N, K = fp8_tensor.shape
|
||||
out = torch.zeros(L, N, K, dtype=output_dtype, device=fp8_tensor.device)
|
||||
n_blocks_n = math.ceil(N / group_n)
|
||||
n_blocks_k = math.ceil(K / group_k)
|
||||
for l_idx in range(L):
|
||||
for bn in range(n_blocks_n):
|
||||
for bk in range(n_blocks_k):
|
||||
n_start = bn * group_n
|
||||
n_end = min(n_start + group_n, N)
|
||||
k_start = bk * group_k
|
||||
k_end = min(k_start + group_k, K)
|
||||
out[l_idx, n_start:n_end, k_start:k_end] = (
|
||||
fp8_tensor[l_idx, n_start:n_end, k_start:k_end].float()
|
||||
* scale[l_idx, bn, bk].float()
|
||||
).to(output_dtype)
|
||||
return out
|
||||
padded = F.pad(
|
||||
fp8_tensor.float(),
|
||||
(0, n_blocks_k * group_k - K, 0, n_blocks_n * group_n - N),
|
||||
)
|
||||
blocks = padded.view(L, n_blocks_n, group_n, n_blocks_k, group_k)
|
||||
out = (blocks * scale.float()[:, :, None, :, None]).to(output_dtype)
|
||||
out = out.view(L, n_blocks_n * group_n, n_blocks_k * group_k)
|
||||
return out[:, :N, :K].contiguous()
|
||||
else:
|
||||
raise ValueError(f"Unsupported tensor ndim: {fp8_tensor.ndim}")
|
||||
|
||||
@@ -519,45 +501,22 @@ def generate_fp8_expand_data(
|
||||
# shared across slices. Compute shared scale across slices, then quantize.
|
||||
# First compute per-token-per-block scale across all slices
|
||||
n_blocks_k = math.ceil(rank / group_k)
|
||||
a_scale = torch.zeros(
|
||||
total_tokens, n_blocks_k, dtype=torch.float32, device=device
|
||||
padded = F.pad(inputs_bf16.float(), (0, n_blocks_k * group_k - rank))
|
||||
blocks = padded.view(nslices, total_tokens, n_blocks_k, group_k)
|
||||
# Take the block amax across all slices so every slice shares the scale.
|
||||
a_scale = blocks.abs().amax(dim=-1).clamp(min=1e-12).amax(dim=0) / FP8_MAX
|
||||
fp8_blocks = (
|
||||
(blocks / a_scale[None, :, :, None]).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
|
||||
)
|
||||
inputs_fp8 = fp8_blocks.view(nslices, total_tokens, -1)[
|
||||
:, :, :rank
|
||||
].contiguous()
|
||||
inputs_dequant = (
|
||||
(fp8_blocks.float() * a_scale[None, :, :, None])
|
||||
.to(dtype)
|
||||
.view(nslices, total_tokens, -1)[:, :, :rank]
|
||||
.contiguous()
|
||||
)
|
||||
for m in range(total_tokens):
|
||||
for bk in range(n_blocks_k):
|
||||
k_start = bk * group_k
|
||||
k_end = min(k_start + group_k, rank)
|
||||
# Max across all slices for this token and block
|
||||
block_amax = torch.tensor(0.0, device=device)
|
||||
for s in range(nslices):
|
||||
block = inputs_bf16[s, m, k_start:k_end].float()
|
||||
block_amax = torch.max(
|
||||
block_amax, block.abs().max().clamp(min=1e-12)
|
||||
)
|
||||
a_scale[m, bk] = (block_amax / FP8_MAX).to(torch.float32)
|
||||
|
||||
# Quantize all slices with the shared scale
|
||||
inputs_fp8_list = []
|
||||
inputs_dequant_list = []
|
||||
for s in range(nslices):
|
||||
slice_2d = inputs_bf16[s] # (total_tokens, rank)
|
||||
fp8_slice = torch.zeros_like(slice_2d, dtype=FP8_DTYPE)
|
||||
dequant_slice = torch.zeros_like(slice_2d)
|
||||
for m in range(total_tokens):
|
||||
for bk in range(n_blocks_k):
|
||||
k_start = bk * group_k
|
||||
k_end = min(k_start + group_k, rank)
|
||||
block = slice_2d[m, k_start:k_end].float()
|
||||
s_val = a_scale[m, bk]
|
||||
fp8_slice[m, k_start:k_end] = (
|
||||
(block / s_val).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
|
||||
)
|
||||
dequant_slice[m, k_start:k_end] = (
|
||||
fp8_slice[m, k_start:k_end].float() * s_val.float()
|
||||
).to(dtype)
|
||||
inputs_fp8_list.append(fp8_slice)
|
||||
inputs_dequant_list.append(dequant_slice)
|
||||
inputs_fp8 = torch.stack(inputs_fp8_list, dim=0)
|
||||
inputs_dequant = torch.stack(inputs_dequant_list, dim=0)
|
||||
elif quant_mode == "per_tensor":
|
||||
# Per-tensor: kernel loads a single scalar from a_scale_ptr
|
||||
inputs_fp8_2d, a_scale = quantize_to_fp8_per_tensor(inputs_2d_all)
|
||||
|
||||
Reference in New Issue
Block a user