diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 55a6a6ff9f..e0e4388d6c 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3814,6 +3814,45 @@ steps: not +quant_fp8 and not +rms_norm and DeepSeek-Coder and ROCM_AITER_MLA' +- label: ":amd: (MI355) DSv4-Flash Disaggregated DP EP" # TBD + timeout_in_minutes: 85 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_8 + num_gpus: 8 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - requirements/kv_connectors_rocm.txt + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/models/deepseek_v4/ + - vllm/v1/attention/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + # A mirror does not inherit the parent env, so it is repeated here in + # the parent's order. The layouts must stay empty: unset, the script + # defaults to HND, which the V4 indexer rejects. + - >- + VLLM_ROCM_USE_AITER=1 + ENABLE_HMA_FLAG=1 + DP_EP=1 + GPU_MEMORY_UTILIZATION=0.85 + PREFILLER_TP_SIZE=4 + DECODER_TP_SIZE=4 + PREFILL_BLOCK_SIZE=256 + DECODE_BLOCK_SIZE=256 + MODEL_NAMES=deepseek-ai/DeepSeek-V4-Flash + VLLM_ENGINE_READY_TIMEOUT_S=1800 + VLLM_SERVE_EXTRA_ARGS=--trust-remote-code,--kv-cache-dtype,fp8 + PREFILLER_KV_LAYOUT= + DECODER_KV_LAYOUT= + bash v1/kv_connector/nixl_integration/run_accuracy_test.sh + #-------------------------------------------------------- mi355 ยท entrypoints --------------------------------------------------------# - label: ":amd: (MI355) Entrypoints Integration (API Server) %N" diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 1258336009..c8ba471ebd 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -337,6 +337,44 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh + mirror: + amd: + label: ":amd: (MI355) DSv4-Flash Disaggregated DP EP" + dind: false + device: mi355_8 + timeout_in_minutes: 85 + depends_on: + - image-build-amd + source_file_dependencies: + - requirements/kv_connectors_rocm.txt + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/models/deepseek_v4/ + - vllm/v1/attention/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + # A mirror does not inherit the parent env, so it is repeated here in + # the parent's order. The layouts must stay empty: unset, the script + # defaults to HND, which the V4 indexer rejects. + - >- + VLLM_ROCM_USE_AITER=1 + ENABLE_HMA_FLAG=1 + DP_EP=1 + GPU_MEMORY_UTILIZATION=0.85 + PREFILLER_TP_SIZE=4 + DECODER_TP_SIZE=4 + PREFILL_BLOCK_SIZE=256 + DECODE_BLOCK_SIZE=256 + MODEL_NAMES=deepseek-ai/DeepSeek-V4-Flash + VLLM_ENGINE_READY_TIMEOUT_S=1800 + VLLM_SERVE_EXTRA_ARGS=--trust-remote-code,--kv-cache-dtype,fp8 + PREFILLER_KV_LAYOUT= + DECODER_KV_LAYOUT= + bash v1/kv_connector/nixl_integration/run_accuracy_test.sh - label: ":nvidia: (H200) Kimi-Linear-48B-A3B Disaggregated DP EP" key: kimi-linear-disaggregated diff --git a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py index a4bfa9ab83..f2999bdb13 100644 --- a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py +++ b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py @@ -13,6 +13,7 @@ mid-decode (silent corruption of an unrelated request). from collections import defaultdict from threading import Event, Lock +from types import SimpleNamespace from unittest.mock import MagicMock, patch import numpy as np @@ -120,16 +121,16 @@ def test_local_descriptors_follow_each_region_pool_capacity(): assert descriptors[:, 0].tolist() == [100, 116, 1000, 1016, 1032] -@pytest.mark.cpu_test -@pytest.mark.parametrize("push_pp", [False, True]) -def test_overlaid_transfer_groups_share_region_geometry(push_pp): - """Groups overlaid on one allocation share its transfer region.""" - import msgspec +def _register_overlaid_mla_worker( + *, push_pp: bool = False, tail_bytes: int = 0, page_covers_view: bool = True +): + """Register two MLA layers overlaid on one allocation. + ``tail_bytes`` leaves spare bytes past the last block, the way an allocation + rounded up to a page boundary does. Clearing ``page_covers_view`` narrows + each layer's view below its page, leaving the block interior non-contiguous. + """ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - NixlAgentMetadata, - ) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( NixlPushConnectorWorker, ) @@ -152,10 +153,12 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp): ) page_size = spec.page_size_bytes block_stride = 2 * page_size - backing = torch.zeros(num_blocks, block_stride, dtype=torch.uint8) + view_size = page_size if page_covers_view else page_size - 8 + allocation = torch.zeros(num_blocks * block_stride + tail_bytes, dtype=torch.uint8) + backing = allocation[: num_blocks * block_stride].view(num_blocks, block_stride) caches = { - "layer.0": backing[:, :page_size], - "layer.1": backing[:, :page_size], + "layer.0": backing[:, :view_size], + "layer.1": backing[:, :view_size], } groups = [KVCacheGroupSpec([layer_name], spec) for layer_name in caches] @@ -203,14 +206,9 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp): worker._logical_num_blocks = num_blocks worker.region_mem_types = [] worker.region_group_ids = [] - worker.region_mem_types = [] worker._mixed_mem_types = False worker.region_names = [] worker.region_num_blocks = [] - worker._mixed_mem_types = False - worker._desc_is_dram_by_block_size = {} - worker._desc_pos_by_block_size = {} - worker._dram_src_handles_by_block_size = {} worker._transfer_layer_names = () worker._transfer_layer_region_indices = () worker._transfer_layer_group_ids = () @@ -231,7 +229,7 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp): num_blocks=num_blocks, kv_cache_tensors=[ KVCacheTensor( - size=backing.nbytes, + size=allocation.nbytes, layers=[name], layer_stride=page_size, block_stride=block_stride, @@ -241,21 +239,43 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp): kv_cache_groups=groups, ) - transfer_topology = MagicMock() - with ( - patch.object(bw, "TransferTopology", return_value=transfer_topology), + patch.object(bw, "TransferTopology", return_value=MagicMock()), patch.object(bw, "compute_nixl_compatibility_hash", return_value="hash"), ): worker.register_kv_caches(caches) + return SimpleNamespace( + worker=worker, + allocation=allocation, + num_blocks=num_blocks, + block_stride=block_stride, + ) + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("push_pp", [False, True]) +def test_overlaid_transfer_groups_share_region_geometry(push_pp): + """Groups overlaid on one allocation share its transfer region.""" + import msgspec + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + NixlAgentMetadata, + ) + + registered = _register_overlaid_mla_worker(push_pp=push_pp) + worker = registered.worker + allocation = registered.allocation + num_blocks = registered.num_blocks + block_stride = registered.block_stride + assert worker.region_group_ids == [-1] assert worker.block_stride_per_layer == [block_stride] assert worker.nixl_wrapper.registered[0][0] == [ - (backing.data_ptr(), backing.nbytes, 0, "") + (allocation.data_ptr(), allocation.nbytes, 0, "") ] expected_addrs = [ - backing.data_ptr() + block * block_stride for block in range(num_blocks) + allocation.data_ptr() + block * block_stride for block in range(num_blocks) ] num_desc_regions = 2 if push_pp else 1 assert worker.src_blocks_data[:, 0].tolist() == expected_addrs * num_desc_regions @@ -275,6 +295,39 @@ def test_overlaid_transfer_groups_share_region_geometry(push_pp): assert worker._block_ids_by_region(([0], [2]), worker.region_group_ids) == [[0, 2]] +def _descriptor_geometry(registered) -> dict: + """The geometry handed to NIXL, addressed relative to the allocation.""" + worker = registered.worker + base = registered.allocation.data_ptr() + return { + "block_len_per_layer": list(worker.block_len_per_layer), + "block_stride_per_layer": list(worker.block_stride_per_layer), + "desc_offsets": [addr - base for addr in worker.src_blocks_data[:, 0].tolist()], + "desc_lens": worker.src_blocks_data[:, 1].tolist(), + } + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "page_covers_view", [True, False], ids=["dense-page", "narrow-view"] +) +def test_registration_ignores_a_sub_block_padding_tail(page_covers_view): + """A tail shorter than one block must not move a single descriptor. + + An allocation rounded up to a page boundary keeps a few spare bytes past the + last block, which cannot hold another block. The narrow view is the branch + DeepSeek-V4-Flash's sliding-window cache lands in. + """ + unpadded = _register_overlaid_mla_worker(page_covers_view=page_covers_view) + # More spare bytes than blocks, so dividing the padded length by the block + # count overshoots the stride and would shift every descriptor. + padded = _register_overlaid_mla_worker( + tail_bytes=2 * unpadded.num_blocks, page_covers_view=page_covers_view + ) + + assert _descriptor_geometry(padded) == _descriptor_geometry(unpadded) + + def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blocks): """Build a real pull worker with a hybrid MLA + 2xKDA HMA layout.""" from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 0f13a265df..b0257de2e5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -1567,7 +1567,13 @@ class NixlBaseConnectorWorker: block_stride = physical_page_size else: block_stride = cache.stride(0) * cache.element_size() - storage_is_block_major = num_blocks * block_stride == registration_len + # A connector that registers the backing as one memory region + # may need its length rounded up to a page boundary. Such a tail + # is shorter than one block, so it cannot make the storage + # segmented. + storage_is_block_major = ( + 0 <= registration_len - num_blocks * block_stride < block_stride + ) # A view with a contiguous page can own a strided region even # when sibling layers share the same backing allocation. page_contiguous = ( @@ -1589,7 +1595,9 @@ class NixlBaseConnectorWorker: ): # TODO(Lucas): handle TP slicing for packed_storage; for now # restrict to MLA (DSv4) where kv is replicated. - storage_block_len = registration_len // num_blocks + # The complete storage row is one block stride. Dividing the + # registered length instead would fold the padding tail in. + storage_block_len = block_stride region_specs = [ (registration_base, storage_block_len, storage_block_len) ] diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index 6c2a05a626..ff6d1a5a6f 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -412,8 +412,8 @@ def allocate_kv_cache( warmup_rocm_skinny_gemm_workspaces(device) # Pad to the page granularity MoRIIO needs to register the shared # backing as a single RDMA memory region. Other platforms keep the - # exact-size allocation: NIXL and SimpleCPUOffload rely on - # storage.nbytes() matching the logical KV size (see #53974). + # exact-size allocation (see #53974), so anything reading + # storage.nbytes() has to tolerate the tail on ROCm alone. page_size = 4096 buf_size = ((raw_size + page_size - 1) // page_size) * page_size else: