diff --git a/tests/lora/test_fused_moe_lora_kernel.py b/tests/lora/test_fused_moe_lora_kernel.py index a70c543473..444b609492 100644 --- a/tests/lora/test_fused_moe_lora_kernel.py +++ b/tests/lora/test_fused_moe_lora_kernel.py @@ -22,6 +22,10 @@ from vllm.platforms import current_platform from vllm.utils.network_utils import get_open_port from vllm.utils.torch_utils import set_random_seed +# The tensor-parallel cases build their distributed environment inside processes +# spawned by torch.multiprocessing.spawn, so this process never has one to tear down. +pytestmark = pytest.mark.skip_global_cleanup + @pytest.fixture(autouse=True) def reset_device(reset_default_device): diff --git a/tests/lora/test_layers.py b/tests/lora/test_layers.py index ab09514274..bce91510d4 100644 --- a/tests/lora/test_layers.py +++ b/tests/lora/test_layers.py @@ -56,14 +56,19 @@ TOLERANCES = { torch.bfloat16: (3e-2, 2e-2), } -pytestmark = pytest.mark.skipif( - not ( - current_platform.is_cuda_alike() - or current_platform.is_cpu() - or current_platform.is_xpu() +pytestmark = [ + pytest.mark.skipif( + not ( + current_platform.is_cuda_alike() + or current_platform.is_cpu() + or current_platform.is_xpu() + ), + reason="Backend not supported", ), - reason="Backend not supported", -) + # Tests here either take dist_init, which tears the distributed environment + # down itself, or never build one, so the global cleanup only repeats it. + pytest.mark.skip_global_cleanup, +] DEVICE_TYPE = current_platform.device_type DEVICES = ( diff --git a/tests/lora/test_punica_ops.py b/tests/lora/test_punica_ops.py index f94b54d9fb..9e1f99b3f6 100644 --- a/tests/lora/test_punica_ops.py +++ b/tests/lora/test_punica_ops.py @@ -15,6 +15,14 @@ from .utils import PunicaTensors, assert_close, generate_data_for_nslices DEVICE_TYPE = current_platform.device_type +# On XPU, oneDNN/oneMKL can return wrong results for these reference matmuls +# after many Triton kernel launches, so the reference stays on CPU there. +_REF_ON_CPU = current_platform.is_xpu() + + +def _to_ref_device(tensor: torch.Tensor) -> torch.Tensor: + return tensor.cpu() if _REF_ON_CPU else tensor + @pytest.fixture(autouse=True) def reset_device(reset_default_device): @@ -33,10 +41,10 @@ def dynamo_reset(): yield -def _cpu_bgmv_shrink( +def _bgmv_shrink( inputs, lora_weight, output, seq_len_tensor, lora_indices, scaling=1.0 ): - """Memory-efficient shrink reference: per-LoRA matmul loop on CPU. + """Memory-efficient shrink reference: per-LoRA matmul loop. output[mask] = scaling * inputs[mask] @ weight.T""" exploded = torch.repeat_interleave(lora_indices, seq_len_tensor) for lid in exploded.unique(): @@ -48,7 +56,7 @@ def _cpu_bgmv_shrink( output[mask] = scaling * (inp @ w.T) -def _cpu_bgmv_expand( +def _bgmv_expand( inputs, lora_weight, output, @@ -57,7 +65,7 @@ def _cpu_bgmv_expand( offset=0, add_inputs=False, ): - """Memory-efficient expand reference: per-LoRA matmul loop on CPU. + """Memory-efficient expand reference: per-LoRA matmul loop. output[mask, offset:offset+n] (+)= inputs[mask] @ weight.T""" exploded = torch.repeat_interleave(lora_indices, seq_len_tensor) for lid in exploded.unique(): @@ -88,21 +96,22 @@ def sgmv_shrink_for_nslices( num_tokens: int, scaling: float, ): - """CPU reference for sgmv_shrink using per-LoRA matmul loop.""" - inp_cpu = inputs_tensor.cpu() - seq_cpu = seq_len_tensor.cpu() - idx_cpu = prompt_lora_mapping.cpu() - out_cpu = out_tensor.cpu() + """Reference for sgmv_shrink using per-LoRA matmul loop.""" + inputs = _to_ref_device(inputs_tensor) + seq_len = _to_ref_device(seq_len_tensor) + mapping = _to_ref_device(prompt_lora_mapping) + out = _to_ref_device(out_tensor) for index in range(nslices): - _cpu_bgmv_shrink( - inp_cpu, - lora_weights_lst[index].cpu(), - out_cpu[index], - seq_cpu, - idx_cpu, + _bgmv_shrink( + inputs, + _to_ref_device(lora_weights_lst[index]), + out[index], + seq_len, + mapping, scaling=scaling, ) - out_tensor.copy_(out_cpu) + if _REF_ON_CPU: + out_tensor.copy_(out) def sgmv_expand_for_nslices( @@ -119,21 +128,22 @@ def sgmv_expand_for_nslices( num_tokens: int, add_inputs: bool, ) -> None: - """CPU reference for sgmv_expand using per-LoRA matmul loop.""" - seq_cpu = seq_len_tensor.cpu() - idx_cpu = prompt_lora_mapping.cpu() - out_cpu = out_tensor.cpu() + """Reference for sgmv_expand using per-LoRA matmul loop.""" + seq_len = _to_ref_device(seq_len_tensor) + mapping = _to_ref_device(prompt_lora_mapping) + out = _to_ref_device(out_tensor) for index in range(nslices): - _cpu_bgmv_expand( - inputs_tensor[index].cpu(), - lora_weights_lst[index].cpu(), - out_cpu, - seq_cpu, - idx_cpu, + _bgmv_expand( + _to_ref_device(inputs_tensor[index]), + _to_ref_device(lora_weights_lst[index]), + out, + seq_len, + mapping, offset=hidden_size * index, add_inputs=add_inputs, ) - out_tensor.copy_(out_cpu) + if _REF_ON_CPU: + out_tensor.copy_(out) _dict_lock = Lock() diff --git a/tests/lora/test_punica_ops_fp8.py b/tests/lora/test_punica_ops_fp8.py index 3e7fe7b275..e447cef9d8 100644 --- a/tests/lora/test_punica_ops_fp8.py +++ b/tests/lora/test_punica_ops_fp8.py @@ -17,6 +17,7 @@ from threading import Lock import pytest import torch +import torch.nn.functional as F import vllm.lora.ops.torch_ops as torch_ops import vllm.lora.ops.triton_ops as triton_ops @@ -31,6 +32,8 @@ from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed +pytestmark = pytest.mark.skip_global_cleanup + DEVICE_TYPE = current_platform.device_type DEVICES = [f"{DEVICE_TYPE}:{0}"] SEED = [0] @@ -206,43 +209,32 @@ def quantize_to_fp8_blockwise( if tensor.ndim == 2: M, K = tensor.shape n_blocks_k = math.ceil(K / group_k) - scale = torch.zeros(M, n_blocks_k, dtype=torch.float32, device=tensor.device) - fp8_tensor = torch.zeros_like(tensor, dtype=FP8_DTYPE) - for m in range(M): - for bk in range(n_blocks_k): - k_start = bk * group_k - k_end = min(k_start + group_k, K) - block = tensor[m, k_start:k_end].float() - amax = block.abs().max().clamp(min=1e-12) - s = (amax / FP8_MAX).to(torch.float32) - scale[m, bk] = s - fp8_tensor[m, k_start:k_end] = ( - (block / s).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE) - ) - return fp8_tensor, scale + # Zero padding leaves the per-block amax unchanged, so the padded + # columns are dropped again after quantizing. + padded = F.pad(tensor.float(), (0, n_blocks_k * group_k - K)) + blocks = padded.view(M, n_blocks_k, group_k) + scale = blocks.abs().amax(dim=-1).clamp(min=1e-12) / FP8_MAX + fp8_tensor = ( + (blocks / scale.unsqueeze(-1)).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE) + ) + return fp8_tensor.view(M, -1)[:, :K].contiguous(), scale elif tensor.ndim == 3: L, N, K = tensor.shape n_blocks_n = math.ceil(N / group_n) n_blocks_k = math.ceil(K / group_k) - scale = torch.zeros( - L, n_blocks_n, n_blocks_k, dtype=torch.float32, device=tensor.device + padded = F.pad( + tensor.float(), + (0, n_blocks_k * group_k - K, 0, n_blocks_n * group_n - N), ) - fp8_tensor = torch.zeros_like(tensor, dtype=FP8_DTYPE) - for li in range(L): - for bn in range(n_blocks_n): - for bk in range(n_blocks_k): - n_start = bn * group_n - n_end = min(n_start + group_n, N) - k_start = bk * group_k - k_end = min(k_start + group_k, K) - block = tensor[li, n_start:n_end, k_start:k_end].float() - amax = block.abs().max().clamp(min=1e-12) - s = (amax / FP8_MAX).to(torch.float32) - scale[li, bn, bk] = s - fp8_tensor[li, n_start:n_end, k_start:k_end] = ( - (block / s).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE) - ) - return fp8_tensor, scale + blocks = padded.view(L, n_blocks_n, group_n, n_blocks_k, group_k) + scale = blocks.abs().amax(dim=(2, 4)).clamp(min=1e-12) / FP8_MAX + fp8_tensor = ( + (blocks / scale[:, :, None, :, None]) + .clamp(FP8_MIN, FP8_MAX) + .to(FP8_DTYPE) + .view(L, n_blocks_n * group_n, n_blocks_k * group_k) + ) + return fp8_tensor[:, :N, :K].contiguous(), scale else: raise ValueError(f"Unsupported tensor ndim: {tensor.ndim}") @@ -301,33 +293,23 @@ def dequantize_fp8_blockwise( """Dequantize FP8 tensor with block-wise scale back to output_dtype.""" if fp8_tensor.ndim == 2: M, K = fp8_tensor.shape - out = torch.zeros(M, K, dtype=output_dtype, device=fp8_tensor.device) n_blocks_k = math.ceil(K / group_k) - for m in range(M): - for bk in range(n_blocks_k): - k_start = bk * group_k - k_end = min(k_start + group_k, K) - out[m, k_start:k_end] = ( - fp8_tensor[m, k_start:k_end].float() * scale[m, bk].float() - ).to(output_dtype) - return out + padded = F.pad(fp8_tensor.float(), (0, n_blocks_k * group_k - K)) + blocks = padded.view(M, n_blocks_k, group_k) + out = (blocks * scale.float().unsqueeze(-1)).to(output_dtype) + return out.view(M, -1)[:, :K].contiguous() elif fp8_tensor.ndim == 3: L, N, K = fp8_tensor.shape - out = torch.zeros(L, N, K, dtype=output_dtype, device=fp8_tensor.device) n_blocks_n = math.ceil(N / group_n) n_blocks_k = math.ceil(K / group_k) - for l_idx in range(L): - for bn in range(n_blocks_n): - for bk in range(n_blocks_k): - n_start = bn * group_n - n_end = min(n_start + group_n, N) - k_start = bk * group_k - k_end = min(k_start + group_k, K) - out[l_idx, n_start:n_end, k_start:k_end] = ( - fp8_tensor[l_idx, n_start:n_end, k_start:k_end].float() - * scale[l_idx, bn, bk].float() - ).to(output_dtype) - return out + padded = F.pad( + fp8_tensor.float(), + (0, n_blocks_k * group_k - K, 0, n_blocks_n * group_n - N), + ) + blocks = padded.view(L, n_blocks_n, group_n, n_blocks_k, group_k) + out = (blocks * scale.float()[:, :, None, :, None]).to(output_dtype) + out = out.view(L, n_blocks_n * group_n, n_blocks_k * group_k) + return out[:, :N, :K].contiguous() else: raise ValueError(f"Unsupported tensor ndim: {fp8_tensor.ndim}") @@ -519,45 +501,22 @@ def generate_fp8_expand_data( # shared across slices. Compute shared scale across slices, then quantize. # First compute per-token-per-block scale across all slices n_blocks_k = math.ceil(rank / group_k) - a_scale = torch.zeros( - total_tokens, n_blocks_k, dtype=torch.float32, device=device + padded = F.pad(inputs_bf16.float(), (0, n_blocks_k * group_k - rank)) + blocks = padded.view(nslices, total_tokens, n_blocks_k, group_k) + # Take the block amax across all slices so every slice shares the scale. + a_scale = blocks.abs().amax(dim=-1).clamp(min=1e-12).amax(dim=0) / FP8_MAX + fp8_blocks = ( + (blocks / a_scale[None, :, :, None]).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE) + ) + inputs_fp8 = fp8_blocks.view(nslices, total_tokens, -1)[ + :, :, :rank + ].contiguous() + inputs_dequant = ( + (fp8_blocks.float() * a_scale[None, :, :, None]) + .to(dtype) + .view(nslices, total_tokens, -1)[:, :, :rank] + .contiguous() ) - for m in range(total_tokens): - for bk in range(n_blocks_k): - k_start = bk * group_k - k_end = min(k_start + group_k, rank) - # Max across all slices for this token and block - block_amax = torch.tensor(0.0, device=device) - for s in range(nslices): - block = inputs_bf16[s, m, k_start:k_end].float() - block_amax = torch.max( - block_amax, block.abs().max().clamp(min=1e-12) - ) - a_scale[m, bk] = (block_amax / FP8_MAX).to(torch.float32) - - # Quantize all slices with the shared scale - inputs_fp8_list = [] - inputs_dequant_list = [] - for s in range(nslices): - slice_2d = inputs_bf16[s] # (total_tokens, rank) - fp8_slice = torch.zeros_like(slice_2d, dtype=FP8_DTYPE) - dequant_slice = torch.zeros_like(slice_2d) - for m in range(total_tokens): - for bk in range(n_blocks_k): - k_start = bk * group_k - k_end = min(k_start + group_k, rank) - block = slice_2d[m, k_start:k_end].float() - s_val = a_scale[m, bk] - fp8_slice[m, k_start:k_end] = ( - (block / s_val).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE) - ) - dequant_slice[m, k_start:k_end] = ( - fp8_slice[m, k_start:k_end].float() * s_val.float() - ).to(dtype) - inputs_fp8_list.append(fp8_slice) - inputs_dequant_list.append(dequant_slice) - inputs_fp8 = torch.stack(inputs_fp8_list, dim=0) - inputs_dequant = torch.stack(inputs_dequant_list, dim=0) elif quant_mode == "per_tensor": # Per-tensor: kernel loads a single scalar from a_scale_ptr inputs_fp8_2d, a_scale = quantize_to_fp8_per_tensor(inputs_2d_all)