From 138810056093301f4881050fcf2b1786939da387 Mon Sep 17 00:00:00 2001 From: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:46:16 -0700 Subject: [PATCH] [Kernel][LoRA] Add deterministic split-K=8 LoRA shrink for batch invariance (#59377) Signed-off-by: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> --- .buildkite/test_areas/misc.yaml | 4 + docs/features/batch_invariance.md | 1 + .../test_lora_shrink_batch_invariant.py | 99 +++++++++++ vllm/lora/ops/triton_ops/kernel_utils.py | 26 ++- vllm/lora/ops/triton_ops/lora_shrink_op.py | 155 +++++++++++++++++- 5 files changed, 275 insertions(+), 10 deletions(-) create mode 100644 tests/v1/determinism/test_lora_shrink_batch_invariant.py diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 602491e0e0..cbcfaeaea7 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -655,12 +655,14 @@ steps: source_file_dependencies: - vllm/v1/attention - vllm/model_executor/layers + - vllm/lora/ops/triton_ops - tests/v1/determinism/ commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pip install pytest-timeout pytest-forked - pytest -v -s v1/determinism/test_batch_invariance.py - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py + - pytest -v -s v1/determinism/test_lora_shrink_batch_invariant.py - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN mirror: @@ -690,12 +692,14 @@ steps: source_file_dependencies: - vllm/v1/attention - vllm/model_executor/layers + - vllm/lora/ops/triton_ops - tests/v1/determinism/ commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pip install pytest-timeout pytest-forked - pytest -v -s v1/determinism/test_batch_invariance.py - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py + - pytest -v -s v1/determinism/test_lora_shrink_batch_invariant.py - pytest -v -s v1/determinism/test_batch_invariance_vlm.py - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md index 07d6d803a7..e82336d241 100644 --- a/docs/features/batch_invariance.md +++ b/docs/features/batch_invariance.md @@ -153,6 +153,7 @@ When batch invariance is enabled, vLLM: 5. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell), runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out (not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes). +6. Splits the LoRA shrink reduction into 8 fixed K chunks that are summed in a fixed order instead of with atomics !!! warning Batch invariance under tensor parallelism is not yet supported for all-reduces whose size isn't a multiple of 16 bytes, for example a hidden size that isn't a multiple of 8 in fp16/bf16. Such tensors can switch between NCCL and custom all-reduce depending on batch size. All validated models meet this requirement. diff --git a/tests/v1/determinism/test_lora_shrink_batch_invariant.py b/tests/v1/determinism/test_lora_shrink_batch_invariant.py new file mode 100644 index 0000000000..2a7029a603 --- /dev/null +++ b/tests/v1/determinism/test_lora_shrink_batch_invariant.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Batch-invariant LoRA shrink: fixed split-K partials, fixed-order reduction.""" + +import pytest +import torch +from utils import skip_if_not_cuda + +import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op +import vllm.lora.ops.triton_ops.utils as lora_utils +from vllm.lora.ops.triton_ops import LoRAKernelMeta, lora_shrink +from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT +from vllm.utils.torch_utils import set_random_seed + +DEVICE = "cuda" + + +@pytest.fixture(autouse=True) +def batch_invariant(monkeypatch): + monkeypatch.setattr(lora_utils, "is_batch_invariant", True) + monkeypatch.setattr(shrink_op, "is_batch_invariant", True) + lora_utils.get_lora_op_configs.cache_clear() + yield + lora_utils.get_lora_op_configs.cache_clear() + + +def _shrink(inputs, weights, mapping, scaling): + num_tokens = inputs.size(0) + meta = LoRAKernelMeta.make(weights[0].size(0), num_tokens, device=DEVICE) + meta.prepare_tensors(mapping) + out = torch.zeros( + (len(weights), num_tokens, weights[0].size(1)), + dtype=torch.float32, + device=DEVICE, + ) + _LORA_A_PTR_DICT.clear() + args = meta.meta_args(token_nums=num_tokens, specialize_active_lora=False) + lora_shrink(inputs, weights, out, *args, scaling) + return out + + +@skip_if_not_cuda +@pytest.mark.parametrize( + "hidden_size,rank,nslices,dtype", + [ + (2560, 8, 3, torch.bfloat16), + (9728, 16, 1, torch.float16), + (4097, 64, 1, torch.bfloat16), # K tail block + (1024, 8, 1, torch.float16), # K < 8 * BLOCK_K: some splits are empty + ], +) +def test_lora_shrink_matches_reference(hidden_size, rank, nslices, dtype): + set_random_seed(0) + inputs = torch.randn((33, hidden_size), dtype=dtype, device=DEVICE) + weights = [ + torch.randn((2, rank, hidden_size), dtype=dtype, device=DEVICE) + for _ in range(nslices) + ] + mapping = torch.randint(-1, 2, (33,), dtype=torch.int32, device=DEVICE) + out = _shrink(inputs, weights, mapping, 0.5).double().cpu() + + ids, x = mapping.cpu(), inputs.double().cpu() + ref = torch.zeros_like(out) + for s, weight in enumerate(weights): + for lora_id in (0, 1): + rows = ids == lora_id + ref[s, rows] = 0.5 * x[rows] @ weight[lora_id].double().cpu().T + torch.testing.assert_close(out, ref, rtol=5e-3, atol=5e-3) + + +@skip_if_not_cuda +@pytest.mark.parametrize("nslices", [1, 3]) +def test_lora_shrink_invariant_across_batch_sizes(monkeypatch, nslices): + reduce_kernel, reduce_grids = shrink_op._lora_shrink_reduce_kernel, [] + + class RecordReduce: + def __getitem__(self, grid): + reduce_grids.append(grid) + return reduce_kernel[grid] + + monkeypatch.setattr(shrink_op, "_lora_shrink_reduce_kernel", RecordReduce()) + set_random_seed(0) + # K = 2560 is ten BLOCK_K=256 blocks, so all 8 splits contribute. + dtype, hidden_size = torch.bfloat16, 2560 + weights = [ + torch.randn((3, 16, hidden_size), dtype=dtype, device=DEVICE) + for _ in range(nslices) + ] + target = torch.randn((1, hidden_size), dtype=dtype, device=DEVICE) + target_id = torch.zeros(1, dtype=torch.int32, device=DEVICE) + expected = _shrink(target, weights, target_id, 0.7)[:, 0] + for num_tokens in (31, 32, 33, 127, 128, 129): + inputs = torch.randn((num_tokens, hidden_size), dtype=dtype, device=DEVICE) + mapping = torch.randint(-1, 3, (num_tokens,), dtype=torch.int32, device=DEVICE) + pos = num_tokens // 2 + inputs[pos], mapping[pos] = target[0], 0 + out = _shrink(inputs, weights, mapping, 0.7) + assert torch.equal(out[:, pos], expected), num_tokens + assert len(reduce_grids) == 7 diff --git a/vllm/lora/ops/triton_ops/kernel_utils.py b/vllm/lora/ops/triton_ops/kernel_utils.py index fd6ae44428..bc07817ae1 100644 --- a/vllm/lora/ops/triton_ops/kernel_utils.py +++ b/vllm/lora/ops/triton_ops/kernel_utils.py @@ -257,6 +257,7 @@ def do_shrink_kernel( output_d0_stride, output_d1_stride, output_d2_stride, + output_split_stride, scaling, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, @@ -265,6 +266,7 @@ def do_shrink_kernel( SPLIT_K: tl.constexpr, SLICE_NUM: tl.constexpr, USE_GDC: tl.constexpr, + STORE_PARTIALS: tl.constexpr, ): """Given an array of integers that identifies the rows of A, ram, a lora index that identifies which LoRA to use from lora_ptr, lora_index, @@ -324,17 +326,31 @@ def do_shrink_kernel( # Identify the C output pointers to store the results of the accumulator. offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N offset_cm = tl.arange(0, BLOCK_M) - cur_out_ptr = out_ptr if SLICE_NUM == 1 else out_ptr + slice_id * output_d0_stride + if STORE_PARTIALS: + cur_out_ptr = ( + out_ptr + + slice_id.to(tl.int64) * output_d0_stride + + pid_sk.to(tl.int64) * output_split_stride + ) + else: + cur_out_ptr = ( + out_ptr if SLICE_NUM == 1 else out_ptr + slice_id * output_d0_stride + ) c_ptr = ( cur_out_ptr + ram[:, None] * output_d1_stride + offset_cn[None, :] * output_d2_stride ) c_mask = (offset_cm[:, None] < M_LEN) & (offset_cn[None, :] < N) - accumulator *= scaling - # handles write-back with reduction-splitting - if SPLIT_K == 1: + if STORE_PARTIALS: + # Unscaled FP32 partial per K split; the reducer adds the splits in a + # fixed order and applies scaling once. tl.store(c_ptr, accumulator, mask=c_mask) else: - tl.atomic_add(c_ptr, accumulator, mask=c_mask, sem="relaxed") + accumulator *= scaling + # handles write-back with reduction-splitting + if SPLIT_K == 1: + tl.store(c_ptr, accumulator, mask=c_mask) + else: + tl.atomic_add(c_ptr, accumulator, mask=c_mask, sem="relaxed") diff --git a/vllm/lora/ops/triton_ops/lora_shrink_op.py b/vllm/lora/ops/triton_ops/lora_shrink_op.py index ec29065b9a..abe63bd41b 100644 --- a/vllm/lora/ops/triton_ops/lora_shrink_op.py +++ b/vllm/lora/ops/triton_ops/lora_shrink_op.py @@ -13,11 +13,90 @@ from vllm.lora.ops.triton_ops.kernel_utils import do_shrink_kernel from vllm.lora.ops.triton_ops.utils import ( _get_lora_a_ptr, get_lora_op_configs, + is_batch_invariant, supports_pdl, ) from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import direct_register_custom_op +# Batch-invariant shrink splits K into this many fixed partials (no atomics). +_BI_SPLIT_K = 8 + + +@triton.jit +def _lora_shrink_reduce_kernel( + partial_ptr, + out_ptr, + M, + N: tl.constexpr, + token_indices_sorted_by_lora_ids, + num_tokens_per_lora, + lora_token_start_loc, + lora_ids, + scaling, + partial_d0_stride, + partial_d1_stride, + partial_d2_stride, + partial_d3_stride, + output_d0_stride, + output_d1_stride, + output_d2_stride, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + SPLIT_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, +): + cta_n_num = tl.cdiv(N, BLOCK_N) + cta_m_num = tl.cdiv(M, BLOCK_M) + pid_m_n = tl.program_id(axis=0) + num_pid_in_group = GROUP_SIZE_M * cta_n_num + group_id = pid_m_n // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(cta_m_num - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + ((pid_m_n % num_pid_in_group) % group_size_m) + pid_n = (pid_m_n % num_pid_in_group) // group_size_m + + slice_id = tl.program_id(axis=1) + lora_idx = tl.program_id(axis=2) + lora_id = tl.load(lora_ids + lora_idx) + if lora_id == -1: + return + + lora_m_size = tl.load(num_tokens_per_lora + lora_idx) + cta_m_offset = pid_m * BLOCK_M + if cta_m_offset >= lora_m_size: + return + + cta_m_len = min(BLOCK_M, lora_m_size - cta_m_offset) + lora_m_indices_start = tl.load(lora_token_start_loc + lora_idx) + cta_lora_seq_indices = ( + token_indices_sorted_by_lora_ids + lora_m_indices_start + cta_m_offset + ) + offset_m = tl.arange(0, BLOCK_M) % cta_m_len + ram = tl.load(cta_lora_seq_indices + offset_m) + offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + offset_cm = tl.arange(0, BLOCK_M) + mask = (offset_cm[:, None] < cta_m_len) & (offset_n[None, :] < N) + + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for split_id in range(SPLIT_K): + partial_in = ( + partial_ptr + + slice_id.to(tl.int64) * partial_d0_stride + + tl.cast(split_id, tl.int64) * partial_d1_stride + + ram[:, None] * partial_d2_stride + + offset_n[None, :] * partial_d3_stride + ) + accumulator += tl.load(partial_in, mask=mask, other=0.0) + + output = ( + out_ptr + + slice_id * output_d0_stride + + ram[:, None] * output_d1_stride + + offset_n[None, :] * output_d2_stride + ) + tl.store(output, accumulator * scaling, mask=mask) + @triton.jit def _lora_shrink_kernel( @@ -49,7 +128,15 @@ def _lora_shrink_kernel( SLICE_NUM: tl.constexpr, USE_GDC: tl.constexpr, launch_pdl: tl.constexpr, + STORE_PARTIALS: tl.constexpr = False, + output_split_stride=0, + PARTIAL_N: tl.constexpr = 0, + PARTIAL_K: tl.constexpr = 0, ): + if STORE_PARTIALS: + # The split-K partial stage specializes on N and K. + N = PARTIAL_N + K = PARTIAL_K cta_n_num = tl.cdiv(N, BLOCK_N) cta_m_num = tl.cdiv(M, BLOCK_M) @@ -116,6 +203,7 @@ def _lora_shrink_kernel( output_d0_stride, output_d1_stride, output_d2_stride, + output_split_stride, scaling, BLOCK_M, BLOCK_N, @@ -124,6 +212,7 @@ def _lora_shrink_kernel( SPLIT_K, SLICE_NUM, USE_GDC, + STORE_PARTIALS, ) @@ -167,6 +256,23 @@ def _lora_shrink( """ assert no_lora_flag_cpu.numel() == 1 + if inputs.size(0) == 0: + # An empty first call has no work to do. + return + partials: torch.Tensor | None = None + if is_batch_invariant: + # Allocated before the no-LoRA return so profiling sees the scratch. + # Per-call allocation keeps each CUDA graph's scratch in its own pool. + partials = torch.empty( + ( + output_tensor.size(0), + _BI_SPLIT_K, + inputs.size(0), + output_tensor.size(-1), + ), + dtype=torch.float32, + device=output_tensor.device, + ) if no_lora_flag_cpu.item(): # None of the inputs require LoRA. return @@ -213,6 +319,11 @@ def _lora_shrink( NUM_STAGES = kernel_config["num_stages"] NUM_CTAS = kernel_config["num_ctas"] GROUP_SIZE_M = kernel_config.get("group_size_m", 8) + + # Every M must keep the same K partition for batch invariance. + if partials is not None: + BLOCK_K = 256 + SPLIT_K = _BI_SPLIT_K EVEN_K = K % (BLOCK_K * SPLIT_K) == 0 # type: ignore # TODO (varun): This grid formulation maximizes parallelization at the @@ -225,11 +336,24 @@ def _lora_shrink( ) # PDL only works when dual-stream is being used. - use_gdc = supports_pdl(inputs.device) and envs.VLLM_LORA_ENABLE_DUAL_STREAM + use_gdc = ( + supports_pdl(inputs.device) + and envs.VLLM_LORA_ENABLE_DUAL_STREAM + and partials is None + ) + first_out = output_tensor if partials is None else partials + partial_args = {} + if partials is not None: + partial_args = dict( + STORE_PARTIALS=True, + output_split_stride=partials.stride(1), + PARTIAL_N=N, + PARTIAL_K=K, + ) _lora_shrink_kernel[grid]( inputs, lora_ptr_tensor, - output_tensor, + first_out, M, N, K, @@ -243,9 +367,9 @@ def _lora_shrink( lora_strides_d0, lora_strides_d1, lora_strides_d2, - output_tensor.stride(0), - output_tensor.stride(1), - output_tensor.stride(2), + first_out.stride(0), + first_out.stride(-2), + first_out.stride(-1), BLOCK_M, BLOCK_N, BLOCK_K, @@ -258,7 +382,28 @@ def _lora_shrink( num_ctas=NUM_CTAS, num_stages=NUM_STAGES, launch_pdl=use_gdc, + **partial_args, ) + if partials is not None: + _lora_shrink_reduce_kernel[(grid[0] // SPLIT_K, *grid[1:])]( + partials, + output_tensor, + M, + N, + token_indices_sorted_by_lora_ids, + num_tokens_per_lora, + lora_token_start_loc, + lora_ids, + scaling, + *partials.stride(), + *output_tensor.stride(), + BLOCK_M, + BLOCK_N, + SPLIT_K, + GROUP_SIZE_M, + num_warps=NUM_WARPS, + num_stages=NUM_STAGES, + ) return