From 593d5a40339ab0c7b0e6b3efb7540bdeab82161c Mon Sep 17 00:00:00 2001 From: zhanqiuhu <49648934+ZhanqiuHu@users.noreply.github.com> Date: Tue, 12 May 2026 09:53:30 -0400 Subject: [PATCH] [Bugfix] Fix mismatched kernel-per-logical blocks in NIXL HMA transfer (#42097) Signed-off-by: ZhanqiuHu Signed-off-by: Zhanqiu Hu Signed-off-by: NickLucche Co-authored-by: NickLucche --- .../unit/test_nixl_connector_hma.py | 308 +++++++++++++++++- .../kv_connector/v1/nixl/worker.py | 116 +++++-- 2 files changed, 395 insertions(+), 29 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index edc591e060..2edd6a8720 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -93,27 +93,39 @@ def test_logical_to_kernel_block_ids_with_hma(): @pytest.mark.cpu_test @pytest.mark.parametrize( - "group_spec_types,expansion_stride,remote_block_ids,expected_remote_block_ids", + "group_spec_types,remote_physical_per_logical," + "local_physical_per_logical,tp_ratio,remote_block_ids," + "expected_remote_block_ids", [ pytest.param( ("FullAttentionSpec", "SlidingWindowSpec"), 2, + 2, + 1, ([0, 1, 2], [3, 4]), [[0, 1, 2, 3, 4, 5], [6, 7, 8, 9]], id="dense_fa_swa", ), + # Nemotron-3-Nano-30B-A3B 4p1d (P_TP=4, D_TP=1): + # remote_physical_per_logical=34, local_physical_per_logical=66. + # FA logical block 5 → kernel [170..203], block 6 → [204..237]. + # Mamba block unchanged. pytest.param( ("FullAttentionSpec", "MambaSpec"), - 261, - ([0, 1, 2], [10, 11]), - [[0, 1, 261, 262, 522, 523], [10, 11]], + 34, + 66, + -4, + ([5, 6], [2]), + [list(range(170, 238)), [2]], id="mamba_fa_ssm", ), ], ) def test_read_blocks_for_req_expands_remote_ids( group_spec_types, - expansion_stride, + remote_physical_per_logical, + local_physical_per_logical, + tp_ratio, remote_block_ids, expected_remote_block_ids, ): @@ -148,7 +160,7 @@ def test_read_blocks_for_req_expands_remote_ids( resolved_types = tuple(spec_name_to_type[n] for n in group_spec_types) worker = object.__new__(NixlConnectorWorker) - worker._physical_blocks_per_logical_kv_block = 2 + worker._physical_blocks_per_logical_kv_block = local_physical_per_logical has_mamba = any(t is MambaSpec for t in resolved_types) has_swa = any(t is SlidingWindowSpec for t in resolved_types) @@ -159,9 +171,11 @@ def test_read_blocks_for_req_expands_remote_ids( remote_engine_id = "remote-engine" worker.transfer_topo = MagicMock() - worker.transfer_topo.tp_ratio.return_value = 1 + # tp_ratio not exercised (all_source_ranks is empty so no reads run), + # but set for realism. + worker.transfer_topo.tp_ratio.return_value = tp_ratio remote_info = MagicMock() - remote_info.remote_physical_blocks_per_logical = expansion_stride + remote_info.remote_physical_blocks_per_logical = remote_physical_per_logical worker.transfer_topo.get_engine_info.return_value = remote_info worker.use_mla = False @@ -192,6 +206,168 @@ def test_read_blocks_for_req_expands_remote_ids( ) +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "local_physical_per_logical,remote_physical_per_logical," + "local_block_ids,remote_block_ids," + "expected_local,expected_remote", + [ + # 10 kernel blocks of data, local has more logical blocks. + # remote physical_per_logical=10 → 1 logical → 10 kernel blocks + # local physical_per_logical=6 → 2 logical → 12 kernel blocks + # Trim local from 12 to 10. + pytest.param( + 6, + 10, + [list(range(12)), [42]], + [list(range(10)), [42]], + [list(range(10)), [42]], + [list(range(10)), [42]], + id="align_local6_remote10", + ), + # 10 kernel blocks of data, remote has more logical blocks. + # remote physical_per_logical=6 → 2 logical → 12 kernel blocks + # local physical_per_logical=10 → 1 logical → 10 kernel blocks + # Trim remote from 12 to 10. + pytest.param( + 10, + 6, + [list(range(10)), [42]], + [list(range(12)), [42]], + [list(range(10)), [42]], + [list(range(10)), [42]], + id="align_local10_remote6", + ), + ], +) +def test_apply_prefix_caching_mamba_hybrid( + local_physical_per_logical, + remote_physical_per_logical, + local_block_ids, + remote_block_ids, + expected_local, + expected_remote, +): + """_apply_prefix_caching front-trims FA groups to + min(local, remote) for Mamba hybrid models with heterogeneous TP. + """ + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( + NixlConnectorWorker, + ) + from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec + + worker = object.__new__(NixlConnectorWorker) + worker._has_mamba = True + worker._physical_blocks_per_logical_kv_block = local_physical_per_logical + worker._group_spec_types = (FullAttentionSpec, MambaSpec) + worker.kv_cache_config = make_kv_cache_config(block_size=16, mamba_enabled=True) + + aligned_local, aligned_remote = worker._apply_prefix_caching( + local_block_ids, remote_block_ids, remote_physical_per_logical + ) + + assert aligned_local == expected_local, ( + f"Expected local {expected_local}, got {aligned_local}" + ) + assert aligned_remote == expected_remote, ( + f"Expected remote {expected_remote}, got {aligned_remote}" + ) + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "local_physical_per_logical,remote_physical_per_logical," + "remote_fa_blocks,local_fa_blocks,ssm_blocks," + "correct_remote_fa,correct_local_fa", + [ + # 10 kernel blocks of data (640 tokens). + # remote physical_per_logical=10 → 1 logical → 10 kernel [0..9] + # local physical_per_logical=6 → 2 logical → 12 kernel [0..11] + # 1st local logical block cached → suffix [6..11] + # Correct: transfer only uncached suffix tokens (384-639) + # = remote [6,7,8,9] → local [6,7,8,9]. + # Actual (front-trim): remote[:6]=[0..5] → local [6..11]. Wrong. + pytest.param( + 6, + 10, + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + [6, 7, 8, 9, 10, 11], + [42], + [6, 7, 8, 9], + [6, 7, 8, 9], + id="local6_remote10_fail", + ), + # 15 kernel blocks of data (960 tokens). + # remote physical_per_logical=6 → 3 logical → 18 kernel [0..17] + # local physical_per_logical=10 → 2 logical → 20 kernel [0..19] + # 1st local logical block cached → suffix [10..19] + # Correct: transfer only uncached suffix tokens (640-959) + # = remote [10,11,12,13,14] → local [10,11,12,13,14]. + # Actual (front-trim): remote[:10]=[0..9] → local [10..19]. Wrong. + pytest.param( + 10, + 6, + list(range(18)), + list(range(10, 20)), + [42], + [10, 11, 12, 13, 14], + [10, 11, 12, 13, 14], + id="local10_remote6_fail", + ), + ], +) +def test_mismatched_physical_per_logical_fails_with_prefix_caching( + local_physical_per_logical, + remote_physical_per_logical, + remote_fa_blocks, + local_fa_blocks, + ssm_blocks, + correct_remote_fa, + correct_local_fa, +): + """Demonstrate that _apply_prefix_caching front-trims ([:N]) + in the Mamba hybrid path, which fails when prefix caching produces + suffix-only local blocks. + + Prefix caching operates at logical block granularity. When a logical + block is cached locally, the decode side only allocates kernel blocks + for the uncached suffix. The front-trim pairs remote prefix blocks + with local suffix slots — a silent data corruption. + """ + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( + NixlConnectorWorker, + ) + + worker = object.__new__(NixlConnectorWorker) + worker._physical_blocks_per_logical_kv_block = local_physical_per_logical + worker.kv_cache_config = make_kv_cache_config( + block_size=16, + mamba_enabled=True, + ) + worker._has_mamba = True + worker._group_spec_types = tuple( + type(g.kv_cache_spec) for g in worker.kv_cache_config.kv_cache_groups + ) + + local_block_ids = (local_fa_blocks, ssm_blocks) + remote_block_ids = (remote_fa_blocks, ssm_blocks) + + aligned_local, aligned_remote = worker._apply_prefix_caching( + local_block_ids, + remote_block_ids, + remote_physical_per_logical, + ) + + assert ( + aligned_remote[0] != correct_remote_fa or aligned_local[0] != correct_local_fa + ), ( + f"Prefix caching with mismatched physical_per_logical should not " + f"produce correct transfer ids: " + f"remote={aligned_remote[0]}, local={aligned_local[0]}, " + f"correct_remote={correct_remote_fa}, correct_local={correct_local_fa}" + ) + + @pytest.mark.parametrize("model_name, sw_size", [("google/gemma-3-1b-it", 512)]) def test_fewer_blocks_with_hma(monkeypatch, model_name, sw_size): """Test that a prefill instance returns fewer "remote blocks" for the SWA groups @@ -564,3 +740,119 @@ def test_compute_physical_blocks_per_logical(ssm_sizes, block_len, expected_rati ) assert compute_physical_blocks_per_logical(ssm_sizes, block_len) == expected_ratio + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "mamba_enabled,swa_enabled," + "local_physical_per_logical,remote_physical_per_logical," + "logical_block_ids,expected_kernel_block_ids", + [ + # Qwen3.5-0.8B 4P2D (kernel_block_size=64): + # prefill TP=4: logical_block_size=384 → physical_per_logical=6 + # decode TP=2: logical_block_size=640 → physical_per_logical=10 + # FA logical [0] → remote kernel [0..9] (1 * 10) + # SSM logical [10] → unchanged [10] + pytest.param( + True, + False, + 6, + 10, + ([0], [10]), + [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [10]], + id="qwen35_4p2d", + ), + # Qwen3.5-0.8B 2P4D (kernel_block_size=64): + # prefill TP=2: logical_block_size=640 → physical_per_logical=10 + # decode TP=4: logical_block_size=384 → physical_per_logical=6 + # FA logical [0, 1] → remote kernel [0..5, 6..11] (2 * 6) + # SSM logical [10] → unchanged [10] + pytest.param( + True, + False, + 10, + 6, + ([0, 1], [10]), + [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], [10]], + id="qwen35_2p4d", + ), + # Homogeneous TP (kernel_block_size=64): + # both sides: logical_block_size=640 → physical_per_logical=10 + # FA logical [0] → kernel [0..9], SSM unchanged + pytest.param( + True, + False, + 10, + 10, + ([0], [10]), + [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [10]], + id="homo_tp", + ), + # remote physical_per_logical=1: early return, no expansion + pytest.param( + True, + False, + 10, + 1, + ([0, 1, 2], [5]), + [[0, 1, 2], [5]], + id="mamba_remote_physical_per_logical_1", + ), + # Pure FA (no mamba): single group expanded with remote stride + pytest.param( + False, + False, + 2, + 4, + ([0, 1],), + [[0, 1, 2, 3, 4, 5, 6, 7]], + id="pure_fa", + ), + # FA + SWA (no mamba): both groups expanded + pytest.param( + False, + True, + 2, + 3, + ([0, 1], [2, 3]), + [[0, 1, 2, 3, 4, 5], [6, 7, 8, 9, 10, 11]], + id="fa_swa", + ), + ], +) +def test_logical_to_remote_kernel_block_ids( + mamba_enabled, + swa_enabled, + local_physical_per_logical, + remote_physical_per_logical, + logical_block_ids, + expected_kernel_block_ids, +): + """Verify _logical_to_remote_kernel_block_ids uses the remote + physical_per_logical for FA expansion, not the local one. + + This was the root cause of silent accuracy corruption in Qwen3.5 + heterogeneous TP (e.g. 4P2D): the old code used local physical_per_logical + for the expansion arange, producing wrong kernel block indices. + + Qwen3.5-0.8B values verified by verify_conv_split.py (issue #13). + """ + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( + NixlConnectorWorker, + ) + + worker = object.__new__(NixlConnectorWorker) + worker._physical_blocks_per_logical_kv_block = local_physical_per_logical + worker.kv_cache_config = make_kv_cache_config( + block_size=16, + mamba_enabled=mamba_enabled, + swa_enabled=swa_enabled, + ) + + result = worker._logical_to_remote_kernel_block_ids( + logical_block_ids, + remote_physical_per_logical, + ) + assert list(result) == expected_kernel_block_ids, ( + f"Expected {expected_kernel_block_ids}, got {result}" + ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py index f7d46af20e..a5f16aec57 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py @@ -1446,6 +1446,23 @@ class NixlConnectorWorker: tp_ratio < 0 and self.transfer_topo.is_kv_replicated(remote_engine_id) ) + remote_physical_per_logical = ( + nixl_agent_meta.physical_blocks_per_logical_kv_block + ) + if ( + self._has_mamba + and remote_physical_per_logical + != self._physical_blocks_per_logical_kv_block + and self.vllm_config.cache_config.enable_prefix_caching + ): + raise RuntimeError( + "Prefix caching with heterogeneous physical_blocks_per_logical " + "is not supported for Mamba hybrid models. " + f"Local: {self._physical_blocks_per_logical_kv_block}, " + f"Remote: {remote_physical_per_logical}. " + "Disable prefix caching with --no-enable-prefix-caching." + ) + if self._is_hma_required: assert block_size_ratio == 1, ( "HMA does not support different remote block size yet" @@ -2182,15 +2199,10 @@ class NixlConnectorWorker: == len(local_block_ids) == len(self.kv_cache_config.kv_cache_groups) ) - # Partial prefix cache hit: just read uncomputed blocks. - # Skip mamba groups — their blocks represent full state (conv+ssm), - # not per-token data, so trimming would corrupt the transfer. - remote_block_ids = list(remote_block_ids) - for i, remote_group in enumerate(remote_block_ids): - num_local_blocks = len(local_block_ids[i]) - assert num_local_blocks <= len(remote_group) - if num_local_blocks < len(remote_group): - remote_block_ids[i] = remote_group[-num_local_blocks:] + remote_physical_per_logical = remote_info.remote_physical_blocks_per_logical + local_block_ids, remote_block_ids = self._apply_prefix_caching( + local_block_ids, remote_block_ids, remote_physical_per_logical + ) # NOTE (nicolo) With homogeneous TP, each TP worker loads KV from # corresponding rank. With heterogeneous TP, fixing D>P, the D tp @@ -2287,6 +2299,66 @@ class NixlConnectorWorker: for i, group in enumerate(block_ids) ] + def _apply_prefix_caching( + self, + local_block_ids: BlockIds, + remote_block_ids: BlockIds, + remote_physical_per_logical: int, + ) -> tuple[BlockIds, list]: + """Apply prefix caching by trimming local/remote block ID lists. + + For non-Mamba models: end-trim remote to match local count, so that + already-cached prefix blocks are skipped in the transfer. + + For Mamba hybrid (prefix caching not yet supported): front-trim both + to the minimum count to handle kernel block count discrepancies from + logical block rounding in heterogeneous TP. + """ + # Partial prefix cache hit: just read uncomputed blocks. + # Skip mamba groups — their blocks represent full state (conv+ssm), + # not per-token data, so trimming would corrupt the transfer. + remote_block_ids = list(remote_block_ids) + if not self._has_mamba: + for i, remote_group in enumerate(remote_block_ids): + num_local_blocks = len(local_block_ids[i]) + assert num_local_blocks <= len(remote_group) + if num_local_blocks < len(remote_group): + remote_block_ids[i] = remote_group[-num_local_blocks:] + else: + # (NOTE: ZhanqiuHu) Mamba hybrid: no prefix caching support so far.HeteroTP + # can cause different kernel block counts due to logical block rounding. + # Example: 640 prompt tokens, kernel_block_size=64 + # remote physical_per_logical=10, local physical_per_logical=6 + # remote logical ids from kv_transfer_params = [0] + # local logical ids allocated = [0, 1] + # remote kernel blocks: [0..9] (1*10=10) + # local kernel blocks: [0..11] (2*6=12) + # actual data blocks = ceil(640/64) = 10, trim both to 10 + # Vice versa (remote physical_per_logical=6, local=10): + # remote logical ids = [0, 1], local logical ids = [0] + # remote kernel blocks: [0..11] (2*6=12) + # local kernel blocks: [0..9] (1*10=10) + # actual data blocks = ceil(640/64) = 10, trim both to 10 + local_block_ids = list(local_block_ids) + for i, remote_group in enumerate(remote_block_ids): + num_local_blocks = len(local_block_ids[i]) + num_remote_blocks = len(remote_group) + if _is_ssm_spec(self._group_spec_types[i]): + assert num_local_blocks == num_remote_blocks + else: + max_padding = max( + self._physical_blocks_per_logical_kv_block, + remote_physical_per_logical, + ) + assert abs(num_local_blocks - num_remote_blocks) < max_padding, ( + f"Group {i}: |{num_local_blocks} - " + f"{num_remote_blocks}| >= {max_padding}" + ) + num_blocks = min(num_local_blocks, num_remote_blocks) + local_block_ids[i] = local_block_ids[i][:num_blocks] + remote_block_ids[i] = remote_group[:num_blocks] + return local_block_ids, remote_block_ids + def _logical_to_remote_kernel_block_ids( self, block_ids: BlockIds, remote_physical_per_logical: int ) -> BlockIds: @@ -2299,23 +2371,25 @@ class NixlConnectorWorker: Returns: Same structure with FA groups expanded (each logical block L - becomes kernel blocks [L*ratio .. L*ratio + local_ratio - 1]). + becomes kernel blocks [L*remote_physical_per_logical, .. + L*remote_physical_per_logical + + remote_physical_per_logical - 1]). Mamba groups are passed through unchanged. """ - local_ratio = self._physical_blocks_per_logical_kv_block if remote_physical_per_logical == 1: return block_ids - local_arange = np.arange(local_ratio).reshape(1, -1) + remote_arange = np.arange(remote_physical_per_logical).reshape(1, -1) group_specs = self.kv_cache_config.kv_cache_groups - result: list[list[int]] = [] - for i, group in enumerate(block_ids): - if not isinstance(group_specs[i].kv_cache_spec, MambaSpec): - arr = np.array(group).reshape(-1, 1) - expanded = (arr * remote_physical_per_logical + local_arange).flatten() - result.append(expanded.tolist()) - else: - # Mamba blocks are 1:1 logical-to-physical (no expansion). - result.append(group) + result = [ + BlockTable.map_to_kernel_blocks( + np.array(group), + remote_physical_per_logical, + remote_arange, + ).tolist() + if not isinstance(group_specs[i].kv_cache_spec, MambaSpec) + else group + for i, group in enumerate(block_ids) + ] return result def get_backend_aware_kv_block_len(