diff --git a/tests/v1/worker/test_kv_block_zeroer.py b/tests/v1/worker/test_kv_block_zeroer.py index 365e4adace..b212e3ae17 100644 --- a/tests/v1/worker/test_kv_block_zeroer.py +++ b/tests/v1/worker/test_kv_block_zeroer.py @@ -20,9 +20,10 @@ def test_block_ids_are_not_overwritten_while_copy_is_in_flight(): zeroer.device = device zeroer._meta = ( torch.tensor([storage.data_ptr()], dtype=torch.uint64, device=device), - page_size_el, - page_size_el, - 1, + torch.tensor([page_size_el], dtype=torch.int64, device=device), + page_size_el // page_size_el, # max_chunks = 1 + page_size_el, # blk_size + 1, # n_segs ) stream = torch.cuda.Stream() @@ -39,3 +40,49 @@ def test_block_ids_are_not_overwritten_while_copy_is_in_flight(): assert torch.all(storage[1] == 0) assert torch.all(storage[2] == 0) assert torch.all(storage[3] == 1) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_non_uniform_page_sizes(): + """Two segments with different page sizes (e.g. MLA + DSA indexer).""" + device = torch.device("cuda") + num_blocks = 4 + page_size_a = 10496 # int32 elements + page_size_b = 2112 + + storage_a = torch.ones((num_blocks, page_size_a), dtype=torch.int32, device=device) + storage_b = torch.ones((num_blocks, page_size_b), dtype=torch.int32, device=device) + + zeroer = KVBlockZeroer.__new__(KVBlockZeroer) + zeroer.device = device + + seg_page_sizes = [page_size_a, page_size_b] + max_ps = max(seg_page_sizes) + + def largest_power_of_2_divisor(n): + return n & -n + + blk_size = min(min(largest_power_of_2_divisor(ps) for ps in seg_page_sizes), 1024) + + zeroer._meta = ( + torch.tensor( + [storage_a.data_ptr(), storage_b.data_ptr()], + dtype=torch.uint64, + device=device, + ), + torch.tensor(seg_page_sizes, dtype=torch.int64, device=device), + max_ps // blk_size, + blk_size, + 2, + ) + + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + zeroer.zero_block_ids([1, 2]) + stream.synchronize() + + for storage in (storage_a, storage_b): + assert torch.all(storage[0] == 1) + assert torch.all(storage[1] == 0) + assert torch.all(storage[2] == 0) + assert torch.all(storage[3] == 1) diff --git a/vllm/model_executor/warmup/qwen_triton_warmup.py b/vllm/model_executor/warmup/qwen_triton_warmup.py index 9b7d76207b..8cbfa539b7 100644 --- a/vllm/model_executor/warmup/qwen_triton_warmup.py +++ b/vllm/model_executor/warmup/qwen_triton_warmup.py @@ -36,7 +36,8 @@ _FLA_POST_CONV_WARMUP_LENGTHS = (1, 2, 16) @dataclass(frozen=True) class _ZeroKvWarmupConfig: - page_size_el: int + seg_page_sizes: torch.Tensor + max_chunks: int block_size: int n_segs: int @@ -160,9 +161,10 @@ def _zero_kv_warmup_config(runner: object) -> _ZeroKvWarmupConfig | None: if meta is None: return None - _, page_size_el, block_size, n_segs = meta + _, seg_page_sizes, max_chunks, block_size, n_segs = meta return _ZeroKvWarmupConfig( - page_size_el=int(page_size_el), + seg_page_sizes=seg_page_sizes, + max_chunks=int(max_chunks), block_size=int(block_size), n_segs=int(n_segs), ) @@ -185,8 +187,9 @@ def _warm_zero_kv_blocks_kernel( from vllm.v1.worker.utils import _zero_kv_blocks_kernel max_n_blocks = max(_ZERO_KV_N_BLOCKS) + max_page_size = int(config.seg_page_sizes.max().item()) scratch = torch.empty( - max_n_blocks * config.page_size_el, + max_n_blocks * max_page_size, dtype=torch.int32, device=device, ) @@ -198,13 +201,14 @@ def _warm_zero_kv_blocks_kernel( for n_blocks in _ZERO_KV_N_BLOCKS: block_ids = torch.arange(n_blocks, dtype=torch.int64, device=device) - grid = (n_blocks * config.n_segs * (config.page_size_el // config.block_size),) + grid = (n_blocks * config.n_segs * config.max_chunks,) _zero_kv_blocks_kernel[grid]( seg_addrs, + config.seg_page_sizes, block_ids, n_blocks, N_SEGS=config.n_segs, - PAGE_SIZE_EL=config.page_size_el, + MAX_CHUNKS=config.max_chunks, BLOCK_SIZE=config.block_size, ) diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index d83242b160..d00974046f 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -43,10 +43,11 @@ logger = init_logger(__name__) @triton.jit def _zero_kv_blocks_kernel( seg_addrs_ptr, + seg_page_sizes_ptr, block_ids_ptr, n_blocks, N_SEGS: tl.constexpr, - PAGE_SIZE_EL: tl.constexpr, + MAX_CHUNKS: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): """Zero KV cache blocks across all segments in a single launch. @@ -56,25 +57,33 @@ def _zero_kv_blocks_kernel( buffer. For backends where K/V is outermost (block_dim=1) there are two segments per buffer (one for K, one for V). + Segments may have different page sizes (e.g. models with multiple KV + cache groups like MLA + DSA indexer). Each segment's page size is + read from seg_page_sizes_ptr; programs whose chunk_index falls beyond + their segment's page size early-exit. + seg_addrs_ptr holds absolute byte addresses (int64) for each segment, allowing segments to live in different CUDA allocations. Programs are mapped as (block_index, seg_index, chunk_index). """ pid = tl.program_id(0) - chunks = PAGE_SIZE_EL // BLOCK_SIZE - work_per_block = N_SEGS * chunks + work_per_block = N_SEGS * MAX_CHUNKS block_index = pid // work_per_block if block_index >= n_blocks: return remainder = pid % work_per_block - seg_index = remainder // chunks - chunk_index = remainder % chunks + seg_index = remainder // MAX_CHUNKS + chunk_index = remainder % MAX_CHUNKS + page_size_el = tl.load(seg_page_sizes_ptr + seg_index) + if chunk_index >= page_size_el // BLOCK_SIZE: + return block_id = tl.load(block_ids_ptr + block_index) seg_addr = tl.load(seg_addrs_ptr + seg_index) ptr = tl.cast(seg_addr, tl.pointer_type(tl.int32)) offset = ( - block_id.to(tl.int64) * PAGE_SIZE_EL + chunk_index.to(tl.int64) * BLOCK_SIZE + block_id.to(tl.int64) * page_size_el.to(tl.int64) + + chunk_index.to(tl.int64) * BLOCK_SIZE ) cols = tl.arange(0, BLOCK_SIZE).to(tl.int64) tl.store(ptr + offset + cols, tl.zeros([BLOCK_SIZE], dtype=tl.int32)) @@ -104,19 +113,19 @@ class KVBlockZeroer: Block IDs from the scheduler reference logical blocks whose size may differ from the kernel block size (virtual block splitting). - PAGE_SIZE_EL accounts for this ratio so that - ``block_id * PAGE_SIZE_EL`` lands at the correct offset. + Each segment's page_size_el accounts for this ratio so that + ``block_id * page_size_el`` lands at the correct offset. Only AttentionSpec layers are processed; Mamba layers are skipped. """ self.device = device - self._meta: tuple[torch.Tensor, int, int, int] | None = None + self._meta: tuple[torch.Tensor, torch.Tensor, int, int, int] | None = None if runner_only_attn_layers is None: runner_only_attn_layers = set() seen_ptrs: set[int] = set() seg_addrs: list[int] = [] - page_size_el: int | None = None + seg_page_sizes: list[int] = [] for group in attn_groups_iter: spec = group.kv_cache_spec @@ -149,12 +158,6 @@ class KVBlockZeroer: assert cur_bytes % 4 == 0 kernel_block_el = cur_bytes // 4 cur_page_el = kernel_block_el * ratio - if page_size_el is None: - page_size_el = cur_page_el - else: - assert page_size_el == cur_page_el, ( - f"Non-uniform page sizes: {page_size_el} vs {cur_page_el}" - ) block_stride_bytes = cur_bytes outer_dims = [ @@ -166,15 +169,21 @@ class KVBlockZeroer: for outer in iprod(*(range(kv.shape[d]) for d in outer_dims)): off_bytes = sum(i * s for i, s in zip(outer, outer_strides)) seg_addrs.append(dp + off_bytes) + seg_page_sizes.append(cur_page_el) - if not seg_addrs or page_size_el is None: + if not seg_addrs: self._meta = None return - blk_size = min(largest_power_of_2_divisor(page_size_el), 1024) + max_page_size_el = max(seg_page_sizes) + blk_size = min( + min(largest_power_of_2_divisor(ps) for ps in seg_page_sizes), + 1024, + ) self._meta = ( torch.tensor(seg_addrs, dtype=torch.uint64, device=self.device), - page_size_el, + torch.tensor(seg_page_sizes, dtype=torch.int64, device=self.device), + max_page_size_el // blk_size, blk_size, len(seg_addrs), ) @@ -183,16 +192,17 @@ class KVBlockZeroer: """Zero the KV cache memory for the given block IDs.""" if not block_ids or self._meta is None: return - seg_addrs, page_size_el, blk_size, n_segs = self._meta + seg_addrs, seg_page_sizes, max_chunks, blk_size, n_segs = self._meta n_blocks = len(block_ids) idx = async_tensor_h2d(block_ids, device=self.device, dtype=torch.int64) - grid = (n_blocks * n_segs * (page_size_el // blk_size),) + grid = (n_blocks * n_segs * max_chunks,) _zero_kv_blocks_kernel[grid]( seg_addrs, + seg_page_sizes, idx, n_blocks, N_SEGS=n_segs, - PAGE_SIZE_EL=page_size_el, + MAX_CHUNKS=max_chunks, BLOCK_SIZE=blk_size, )