diff --git a/cmake/external_projects/flashkda.cmake b/cmake/external_projects/flashkda.cmake index 5ac4f4b415..fadee58efb 100644 --- a/cmake/external_projects/flashkda.cmake +++ b/cmake/external_projects/flashkda.cmake @@ -13,7 +13,7 @@ else() FetchContent_Declare( flashkda GIT_REPOSITORY https://github.com/vllm-project/FlashKDA.git - GIT_TAG b59532f1f464fbd536272780e30df5bf6a2ccc02 + GIT_TAG 17a037d98da546deb4591e967cf961a43c034d8b GIT_PROGRESS TRUE GIT_SUBMODULES cutlass ) diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md index 7353bd3594..0fad4682fc 100644 --- a/docs/features/batch_invariance.md +++ b/docs/features/batch_invariance.md @@ -144,7 +144,7 @@ When batch invariance is enabled, vLLM: 1. Uses deterministic kernel implementations for attention and other operations 2. Ensures consistent numerical behavior across different batch sizes -3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode) +3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant) 4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell), runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out (not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes). diff --git a/docs/mkdocs/gen_files/generate_metrics.py b/docs/mkdocs/gen_files/generate_metrics.py index b952f53977..4d4fa93cc5 100644 --- a/docs/mkdocs/gen_files/generate_metrics.py +++ b/docs/mkdocs/gen_files/generate_metrics.py @@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [ "path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py", "key": "metrics-nixl", }, + { + "path": "vllm/v1/simple_kv_offload/metrics.py", + "key": "metrics-simple-kv-offload", + }, {"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"}, ] diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md index 305c21478f..8254a7f420 100644 --- a/docs/usage/metrics.md +++ b/docs/usage/metrics.md @@ -45,6 +45,30 @@ The following metrics are exposed: --8<-- "gen:metrics-nixl" +## Simple CPU Offload Connector Metrics + +These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector +is configured (e.g. `--kv-transfer-config='{"kv_connector": +"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config": +{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are +updated once per engine step. + +Caveats to keep in mind when interpreting them: + +- A "completed" store means the write syscalls returned; it is **not** + fsync-durable, and the disk backend's file is process-lifetime scratch, + unlinked at startup and shutdown. +- `save_outcomes_total` classifies eager-mode boundary hand-off stores; + lazy-mode stores are not classified. +- `used_blocks` counts blocks pinned by in-flight transfers or cache hits; + warm cached blocks that are evictable are not counted. Use the + `capacity_blocks` label of `simple_kv_offload_info` as the denominator. +- Counters and gauges are quantized to engine steps. They are reported by the + scheduler process and reflect engine-wide logical block counts, not values + pooled from individual tensor-parallel workers. + +--8<-- "gen:metrics-simple-kv-offload" + ## Model Flops Utilization (MFU) Performance Metrics These metrics are available via `--enable-mfu-metrics`: diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index 8fa1706f18..fbc8586fd3 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -566,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env( assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected -def test_aiter_moe_situv2_syncs_aiter_a4w4_env( - monkeypatch: pytest.MonkeyPatch, -): - """SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides.""" - from vllm._aiter_ops import rocm_aiter_ops - - _assert_aiter_supported() - - with monkeypatch.context() as mp: - mp.delenv("AITER_SITUV2_A8W4", raising=False) - mp.delenv("AITER_SITUV2_A4W4", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1") - _reload_envs() - rocm_aiter_ops.refresh_env_variables() - - import os - - assert os.environ.get("AITER_SITUV2_A4W4") == "1" - assert "AITER_SITUV2_A8W4" not in os.environ - - -def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4( - monkeypatch: pytest.MonkeyPatch, -): - """Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4.""" - from vllm._aiter_ops import rocm_aiter_ops - - _assert_aiter_supported() - - with monkeypatch.context() as mp: - mp.delenv("AITER_SITUV2_A8W4", raising=False) - mp.delenv("AITER_SITUV2_A4W4", raising=False) - mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1") - _reload_envs() - rocm_aiter_ops.refresh_env_variables() - - import os - - assert os.environ.get("AITER_SITUV2_A4W4") == "1" - assert "AITER_SITUV2_A8W4" not in os.environ - - -def test_aiter_moe_situv2_clears_aiter_env_when_disabled( +@pytest.mark.parametrize( + "value,expected_act,expected_env", + [ + # Unset / auto / legacy "1" all mean the a4w4 default. + (None, "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("0", "a16w4", {}), + ("a16w4", "a16w4", {}), + ("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}), + ], +) +def test_aiter_moe_situv2_activation_syncs_aiter_env( + value: str | None, + expected_act: str, + expected_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, ): + """Each SiTUv2 activation choice sets exactly its AITER dispatch env.""" + import os + from vllm._aiter_ops import rocm_aiter_ops _assert_aiter_supported() with monkeypatch.context() as mp: + # Stale AITER overrides must be replaced by the vLLM selection. + mp.setenv("AITER_SITUV2_A8W4", "1") mp.setenv("AITER_SITUV2_A4W4", "1") mp.setenv("VLLM_ROCM_USE_AITER", "1") mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0") + if value is None: + mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False) + else: + mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value) _reload_envs() rocm_aiter_ops.refresh_env_variables() - import os + assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act + assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is ( + expected_act == "a8w4" + ) + assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4") + for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"): + assert os.environ.get(name) == expected_env.get(name) - assert "AITER_SITUV2_A4W4" not in os.environ + +def test_aiter_moe_situv2_rejects_unknown_activation( + monkeypatch: pytest.MonkeyPatch, +): + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8") + _reload_envs() + import vllm.envs as envs + + with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"): + _ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 @pytest.mark.parametrize("moe_padding", [True, False]) diff --git a/tests/kernels/test_fused_inv_rope_fp8_quant.py b/tests/kernels/test_fused_inv_rope_fp8_quant.py index 03d5b7921f..3b5b24fe49 100644 --- a/tests/kernels/test_fused_inv_rope_fp8_quant.py +++ b/tests/kernels/test_fused_inv_rope_fp8_quant.py @@ -21,7 +21,12 @@ Usage: import pytest import torch +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + deepgemm_post_process_fp8_weight_block, +) from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant +from vllm.platforms import current_platform +from vllm.utils.deep_gemm import fp8_einsum # -- Default dimensions matching DeepSeek V3/V4 -------------------------- HEAD_DIM = 512 @@ -33,6 +38,49 @@ FP8_DTYPE = torch.float8_e4m3fn EPS = 1e-10 +@pytest.mark.skipif( + not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103" +) +@pytest.mark.parametrize("groups", [2, 4]) +@pytest.mark.parametrize("tokens", [1, 6, 18, 32]) +def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens): + """Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize.""" + from flashinfer import mxfp8_quantize + + from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL + + torch.manual_seed(0) + n = groups * 1024 + x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128] + positions = torch.randint(0, 128, (tokens,), device="cuda") + rope = make_cos_sin_cache(128, device="cuda") + wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE) + scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float()) + wq, ws = deepgemm_post_process_fp8_weight_block( + wq, scale, (1, 32), False, True, groups + ) + q, sf = _FUSED_WO_A_KERNEL( + x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws + ) + + a, sa = fused_inv_rope_fp8_quant( + x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True + ) + y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16) + fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32)) + ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl") + + def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks] + return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1) + + actual, expected = ( + d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0) + for d, s in ((q, sf), (ref, ref_sf)) + ) + assert (actual - expected).norm() / expected.norm() < 3e-3 + assert not rows(sf)[tokens:].any(), "padded scale rows must be zero" + + # ========================================================================= # Helpers # ========================================================================= diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py index 2f8b80a7c1..517184c0f1 100644 --- a/tests/v1/attention/test_attention_backends.py +++ b/tests/v1/attention/test_attention_backends.py @@ -942,8 +942,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa impl.o_sf_scale = None impl.window_left = -1 impl.sinks = None - impl.cache_config = unittest.mock.Mock() - impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC layer = unittest.mock.Mock( _q_scale=torch.tensor(1.0), @@ -959,6 +957,7 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa q_len_per_req=1, ) attn_metadata = flashinfer_backend.FlashInferMetadata( + kv_cache_layout=KVCacheLayout.LBHNC, num_actual_tokens=2, slot_mapping=torch.empty(0, dtype=torch.int64), q_data_type_prefill=torch.bfloat16, diff --git a/tests/v1/attention/test_group_head_counts.py b/tests/v1/attention/test_group_head_counts.py index 54fe3cfdbc..40d9e2d00f 100644 --- a/tests/v1/attention/test_group_head_counts.py +++ b/tests/v1/attention/test_group_head_counts.py @@ -101,6 +101,7 @@ def test_flash_attention_geometry_comes_from_the_group(): num_kv_heads=2, head_size=64, dtype=torch.bfloat16, + dcp_sharded=True, ) with ( diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 24be16e4a5..c2c86053f0 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -1599,19 +1599,142 @@ def test_project_kv_cache_groups_to_worker(): assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"} -def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only(): - dcp = 8 - full = FullAttentionSpec( - block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32 +@pytest.mark.parametrize("sliding_window", [None, 256]) +@pytest.mark.parametrize("disable_hybrid", [False, True]) +@pytest.mark.parametrize("pcp_size", [1, 4]) +def test_dcp_target_allocates_replicated_draft_independently( + monkeypatch, sliding_window, disable_hybrid, pcp_size +): + """A draft must retain all positions even when the target shards them.""" + from vllm.v1.attention.backends.utils import resolve_kv_cache_layout + + monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False) + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.parallel_config.decode_context_parallel_size = 4 + config.parallel_config.prefill_context_parallel_size = pcp_size + config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hybrid + config.cache_config.block_size = 16 + config.cache_config.kv_cache_layout = None + draft_args = dict( + block_size=16, + num_kv_heads=8, + head_size=64, + dtype=torch.bfloat16, + dcp_sharded=False, ) - mla = new_mla_spec() - mamba = new_mamba_spec() - uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla}) - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1 - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1 + draft = ( + FullAttentionSpec(**draft_args) + if sliding_window is None + else SlidingWindowSpec(**draft_args, sliding_window=sliding_window) + ) + specs = {"target": new_mla_spec(), "draft": draft} + layout = resolve_kv_cache_layout(config, [["LBHNC", "BLHNC"]], specs.values()) + assert layout == KVCacheLayout.BLHNC + groups = get_kv_cache_groups(config, specs) + assert [group.layer_names for group in groups] == [["target"], ["draft"]] + widths = [g.kv_cache_spec.max_num_blocks_per_req(config, 1024) for g in groups] + assert widths == [16, 64] + + cache_config = kv_cache_utils.get_kv_cache_config_from_groups( + config, groups, available_memory=16 * 1024 * 1024 + ) + scheduler_config = generate_scheduler_kv_cache_config([cache_config]) + assert get_max_concurrency_for_kv_cache_config(config, cache_config) > 0 + assert get_max_concurrency_for_kv_cache_config(config, cache_config) == ( + get_max_concurrency_for_kv_cache_config(config, scheduler_config) + ) + manager = KVCacheManager( + scheduler_config, + max_model_len=1024, + hash_block_size=16, + scheduler_block_size=64, + dcp_world_size=4, + pcp_world_size=pcp_size, + enable_caching=True, + ) + assert manager.coordinator.group_block_sizes == (64, 16) + request = make_request("replicated-draft", [1] * 65, block_size=16, hash_fn=sha256) + blocks = manager.allocate_slots(request, 65) + assert blocks is not None + assert [len(group) for group in blocks.blocks] == [2, 5] + manager.cache_blocks(request, 64) + cached_request = make_request( + "cached-draft", [1] * 65, block_size=16, hash_fn=sha256 + ) + cached_blocks, num_cached, _ = manager.get_computed_blocks(cached_request) + assert num_cached == 64 + assert [len(group) for group in cached_blocks.blocks] == [1, 4] + + +@pytest.mark.parametrize("use_mla", [False, True]) +def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla): + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.parallel_config.decode_context_parallel_size = 4 + draft = replace( + new_mla_spec() if use_mla else new_kv_cache_spec(), + block_size=16, + dcp_sharded=False, + ) + merged = type(draft).merge([draft, draft]) + wrapped = UniformTypeKVCacheSpecs.from_specs({"draft": merged}) + assert wrapped is not None and not wrapped.dcp_sharded + assert merged.max_num_blocks_per_req(config, 1024) == 64 + assert merged.max_memory_usage_bytes(config) == 64 * draft.page_size_bytes + assert kv_cache_utils.resolve_dcp_kv_block_size(merged, 4) == 16 + with pytest.raises(AssertionError): + type(draft).merge([draft, replace(draft, dcp_sharded=True)]) + + +@pytest.mark.parametrize("with_draft", [False, True]) +@pytest.mark.parametrize("indexer_alignment", [None, 512]) +def test_sparse_mla_preserves_physical_row_addressing(with_draft, indexer_alignment): + from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view + from vllm.v1.worker.utils import allocate_kv_cache + + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.cache_config.kv_cache_layout = "BLHNC" + config.parallel_config.decode_context_parallel_size = 4 + common = dict(block_size=64, num_kv_heads=1, dtype=torch.uint8) + specs = { + "target": MLAAttentionSpec( + **common, + head_size=576, + state_content_bytes=656, + block_stride_alignment=656, + cache_dtype_str="fp8_ds_mla", + is_index_group_leader=True, + ), + "indexer": MLAAttentionSpec( + **common, + head_size=132, + cache_role=SparseCacheRole.INDEXER, + block_stride_alignment=indexer_alignment, + ), + "draft": SlidingWindowSpec( + block_size=64, + num_kv_heads=64, + head_size=64, + dtype=torch.bfloat16, + sliding_window=2048, + dcp_sharded=False, + ), + } + if not with_draft: + del specs["draft"] + groups = get_kv_cache_groups(config, specs) + cache_config = kv_cache_utils.get_kv_cache_config_from_groups( + config, groups, available_memory=8 * 1024 * 1024 + ) + caches = allocate_kv_cache( + cache_config, torch.device("cpu"), KVCacheLayout.BLHNC, [64] * len(groups) + ) + cache = caches["target"].squeeze(1) + if indexer_alignment is not None: + assert caches["indexer"].stride(0) % indexer_alignment == 0 + rows, stride_rows = flat_kv_row_view(cache, 64) + cache[1, 0].fill_(7) + torch.testing.assert_close(rows[stride_rows], cache[1, 0]) + assert torch.all(rows[stride_rows] == 7) @pytest.mark.parametrize( @@ -3815,6 +3938,8 @@ def test_unify_hybrid_kv_cache_specs(): "layer_1": before_spec_1, "layer_2": before_spec_2, } + kv_cache_spec["draft_layer_1"] = replace(before_spec_1, dcp_sharded=False) + kv_cache_spec["draft_layer_2"] = replace(before_spec_2, dcp_sharded=False) kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec) expected_spec_1 = new_kv_cache_spec(block_size=64) expected_spec_2 = new_kv_cache_spec( @@ -3823,6 +3948,7 @@ def test_unify_hybrid_kv_cache_specs(): assert kv_cache_spec["layer_1"] == expected_spec_1 assert kv_cache_spec["layer_2"] == expected_spec_2 assert kv_cache_spec["layer_2"].page_size_bytes == 64 * 1024 + assert kv_cache_spec["draft_layer_2"] == replace(expected_spec_2, dcp_sharded=False) # 2. has_full_attention and has_chunked_local_attention before_spec_1 = new_kv_cache_spec() @@ -3874,6 +4000,11 @@ def test_unify_hybrid_kv_cache_specs(): with pytest.raises(ValueError): kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec) + # Replicated Mamba state still requires the hybrid cache manager. + kv_cache_spec = {"attention": new_kv_cache_spec(), "mamba": new_mamba_spec()} + with pytest.raises(ValueError): + kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec) + def test_unify_kv_cache_spec_page_size_mamba(): """Regression test for https://github.com/vllm-project/vllm/issues/43626. diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 0a34a6131d..99d3235006 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -1336,12 +1336,12 @@ def make_kv_cache_config_three_types( ) -def test_prefix_cache_hit_uses_per_group_dcp_geometry(): +@pytest.mark.parametrize("draft_sharded", [True, False]) +def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded): """Prefix lookup must use each group's DCP size, not the process-wide one. - Target and draft MLA are both sharded (DCP=8); Mamba stays replicated - (DCP=1). Hits then align to the sharded full-attention block, not the - unsharded page size. + Target MLA is sharded; draft MLA can opt out and Mamba stays replicated. + Hits must align to the target block in either case. """ block_size = 16 dcp = 8 @@ -1362,6 +1362,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(): KVCacheGroupSpec( ["draft_mla"], MLAAttentionSpec( + dcp_sharded=draft_sharded, block_size=block_size, num_kv_heads=1, head_size=1, @@ -1388,10 +1389,10 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(): ) target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers assert target_mgr.dcp_world_size == dcp - assert draft_mgr.dcp_world_size == dcp + assert draft_mgr.dcp_world_size == (dcp if draft_sharded else 1) assert mamba_mgr.dcp_world_size == 1 assert target_mgr.block_size == sharded_block - assert draft_mgr.block_size == sharded_block + assert draft_mgr.block_size == (sharded_block if draft_sharded else block_size) assert mamba_mgr.block_size == block_size hash_fn = sha256 @@ -1407,7 +1408,11 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(): req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn) computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert num_computed_tokens == 2 * sharded_block - assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16] + assert [len(group) for group in computed_blocks.blocks] == [ + 2, + 2 if draft_sharded else 16, + 16, + ] manager.free(req0) manager.free(req1) diff --git a/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py b/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py index 3a474ac203..d2c2ac87b7 100644 --- a/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py +++ b/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py @@ -11,6 +11,7 @@ from vllm.config import CompilationConfig, VllmConfig, replace from vllm.config.kernel import MoEBackend from vllm.engine.arg_utils import EngineArgs from vllm.platforms import current_platform +from vllm.v1.spec_decode.draft_model import DraftModelProposer from ..utils import ( Messages, @@ -196,6 +197,38 @@ def test_draft_model_engine_args_tensor_parallelism(): assert draft_config.quant_config is None +@pytest.mark.cpu_test +@pytest.mark.parametrize("dcp_size", [1, 2]) +def test_draft_model_inherits_target_dcp_config(monkeypatch, dcp_size): + monkeypatch.setattr(current_platform, "device_count", lambda: 16) + target_config = EngineArgs( + model="Qwen/Qwen3-1.7B", + tensor_parallel_size=16, + decode_context_parallel_size=dcp_size, + cp_kv_cache_interleave_size=16, + dcp_comm_backend="ag_rs", + distributed_executor_backend="mp", + speculative_config={ + "model": "Qwen/Qwen3-0.6B", + "method": "draft_model", + "num_speculative_tokens": 3, + }, + ).create_engine_config() + proposer = object.__new__(DraftModelProposer) + proposer.vllm_config = target_config + proposer.speculative_config = target_config.speculative_config + draft_config = proposer._create_draft_vllm_config() + draft_parallel = draft_config.parallel_config + assert draft_parallel.decode_context_parallel_size == dcp_size + assert draft_parallel.cp_kv_cache_interleave_size == 16 + assert draft_parallel.dcp_comm_backend == "ag_rs" + assert ( + draft_parallel.dcp_q_replicate == target_config.parallel_config.dcp_q_replicate + ) + assert draft_parallel is not target_config.parallel_config + assert target_config.parallel_config.decode_context_parallel_size == dcp_size + + def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig: """Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic so we can test it without instantiating a full proposer.""" diff --git a/tests/v1/kv_connector/unit/test_decode_bench_connector.py b/tests/v1/kv_connector/unit/test_decode_bench_connector.py index 693c10d40a..6f518ab53a 100644 --- a/tests/v1/kv_connector/unit/test_decode_bench_connector.py +++ b/tests/v1/kv_connector/unit/test_decode_bench_connector.py @@ -500,7 +500,7 @@ def test_decode_bench_connector_skips_null_padding(): def test_decode_bench_connector_uses_per_group_dcp_block_sizes(): - """DCP scales full-attention blocks but not sliding-window blocks.""" + """DCP scales target blocks but not the replicated draft's window.""" block_size = 16 dcp_world_size = 2 vllm_config = create_vllm_config( @@ -515,6 +515,7 @@ def test_decode_bench_connector_uses_per_group_dcp_block_sizes(): dtype=torch.float32, ) sliding_window = SlidingWindowSpec( + dcp_sharded=False, block_size=block_size, num_kv_heads=1, head_size=1, diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 947e9f6a20..f08213444e 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -290,6 +290,8 @@ class _FakeKVTransferConfig: class _FakeModelConfig: model = "test-model" use_mla = False + enable_sleep_mode = False + enable_cumem_allocator = False def get_num_layers(self, parallel_config) -> int: return 1 @@ -432,6 +434,64 @@ def _patch_worker_runtime( monkeypatch.setattr(worker, "LookupKeyServer", MagicMock()) +def test_create_mem_pool_returns_none_when_unconfigured(): + assert worker.MooncakeStoreWorker._create_mem_pool({}, _FakeModelConfig()) is None + + +def test_create_mem_pool_rejects_unknown_pool(): + with pytest.raises( + ValueError, + match="Unsupported custom_mem_pool='UNKNOWN'", + ): + worker.MooncakeStoreWorker._create_mem_pool( + {"custom_mem_pool": "unknown"}, _FakeModelConfig() + ) + + +@pytest.mark.parametrize( + "conflicting_option", ["enable_sleep_mode", "enable_cumem_allocator"] +) +def test_create_mem_pool_rejects_cumem_conflicts(conflicting_option): + model_config = _FakeModelConfig() + setattr(model_config, conflicting_option, True) + + with pytest.raises(ValueError, match="custom_mem_pool is incompatible"): + worker.MooncakeStoreWorker._create_mem_pool( + {"custom_mem_pool": "NVLINK"}, model_config + ) + + +@pytest.mark.parametrize( + ("pool_type", "allocator_name"), + [("nvlink", "NVLinkAllocator"), ("barex", "BarexAllocator")], +) +def test_create_mem_pool_uses_mooncake_allocator( + monkeypatch, pool_type, allocator_name +): + allocator_cls = MagicMock() + allocator = allocator_cls.get_allocator.return_value + allocator_module = types.ModuleType("mooncake.allocator") + setattr(allocator_module, allocator_name, allocator_cls) + mooncake_module = types.ModuleType("mooncake") + mooncake_module.allocator = allocator_module # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "mooncake", mooncake_module) + monkeypatch.setitem(sys.modules, "mooncake.allocator", allocator_module) + + monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 2) + mem_pool = MagicMock() + mem_pool_ctor = MagicMock(return_value=mem_pool) + monkeypatch.setattr(torch.cuda, "MemPool", mem_pool_ctor) + + result = worker.MooncakeStoreWorker._create_mem_pool( + {"custom_mem_pool": pool_type}, _FakeModelConfig() + ) + + assert result is mem_pool + allocator_cls.get_allocator.assert_called_once_with(torch.device("cuda", 2)) + allocator.allocator.assert_called_once_with() + mem_pool_ctor.assert_called_once_with(allocator.allocator.return_value) + + def test_pool_key_to_string_without_prefix_is_unchanged(): """Default (empty) cache_prefix keeps keys byte-identical to the historical unprefixed format so existing deployments keep their hits.""" diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index 1a85865560..9d04a2ea53 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -68,6 +68,7 @@ class MockConnector(KVConnectorBase_V1): # Override just build_kv_connector_stats mock.build_kv_connector_stats = cls.build_kv_connector_stats mock.get_kv_connector_stats.return_value = None + mock.get_mem_pool_context.return_value = None return mock @classmethod @@ -201,6 +202,36 @@ def mc() -> MultiConnector: return mc +def test_multi_connector_mem_pool_context_none(mc: MultiConnector): + assert mc.get_mem_pool_context() is None + + +def test_multi_connector_forwards_mem_pool_context(mc: MultiConnector): + context = MagicMock() + provider = MagicMock(spec_set=KVConnectorBase_V1) + provider.get_mem_pool_context.return_value = context + mc._connectors = [mc._connectors[0], provider] + + assert mc.get_mem_pool_context() is context + provider.get_mem_pool_context.assert_called_once_with() + + +def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector): + providers = [ + MagicMock(spec_set=KVConnectorBase_V1), + MagicMock(spec_set=KVConnectorBase_V1), + ] + for provider in providers: + provider.get_mem_pool_context.return_value = MagicMock() + mc._connectors = providers + + with pytest.raises( + ValueError, + match="Multiple connectors provide a KV cache memory pool", + ): + mc.get_mem_pool_context() + + # Helper function to compare directories recursively def _compare_directories(dir1: Path, dir2: Path) -> bool: """Compares two directories recursively for identical content.""" @@ -320,6 +351,7 @@ def test_multi_example_connector_consistency(): # First three events are from initialization. Layer hooks run before the # deferred load starts after the forward pass. expected_worker_prefix = [ + "get_mem_pool_context", "register_kv_caches", "set_host_xfer_buffer_ops", "get_handshake_metadata", diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 39f8f88d5c..d8b6fcb5a0 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -7,6 +7,7 @@ import os import queue import tempfile import textwrap +import threading import time import uuid from collections import defaultdict @@ -49,6 +50,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( NixlKVConnectorStats, ) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + RemoteMeta, + ReqMeta, compute_nixl_compatibility_hash, ) from vllm.distributed.kv_transfer.kv_transfer_state import ( @@ -2087,6 +2090,10 @@ def recv_worker(): worker._recving_transfers = defaultdict(list) worker._failed_recv_reqs = queue.Queue() worker._recv_failures = set() + worker._handshake_lock = threading.RLock() + worker._handshake_futures = {} + worker._remote_agents = {} + worker._engine_by_address = {} worker._replicated_pcp_done_sending = set() worker._invalid_block_ids = queue.Queue() worker._pending_recv_notifs = {} @@ -2681,6 +2688,174 @@ def test_engine_ttl_disabled(default_vllm_config, dist_init): assert engine_id in worker.dst_xfer_side_handles +@pytest.mark.cpu_test +class TestPeerReplacement: + @pytest.fixture(autouse=True) + def setup(self, monkeypatch): + from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw + from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend + + config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0}) + config.kv_transfer_config.kv_buffer_device = "cpu" + platform = SimpleNamespace( + device_type="cpu", + discover_numa_topology=lambda: [], + get_nixl_memory_type=lambda: "DRAM", + is_rocm=lambda: False, + ) + with ( + patch.object(bw, "NixlWrapper", FakeNixlWrapper), + patch.object(bw, "current_platform", platform), + patch.object(bw, "get_tensor_model_parallel_rank", return_value=0), + patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1), + patch.object( + bw, "get_current_attn_backends", return_value=[FlashAttentionBackend] + ), + ): + self.worker = FakeNixlConnectorWorker(config, "local", hand_shake_latency=0) + self.nixl = self.worker.nixl_wrapper + self.transport = MagicMock(wraps=self.nixl) + self.worker.nixl_wrapper = self.transport + monkeypatch.setattr( + self.worker._handshake_initiation_executor, + "submit", + lambda *args: Future(), + ) + self._connect("old") + yield + self.worker.shutdown() + + def _connect(self, engine_id="new", host="localhost", port=1234): + future = self.worker._ensure_handshake(engine_id, host, port, 1) + self.worker.REMOTE_ENGINE_ID = engine_id + self.nixl.REMOTE_AGENT_NAME = engine_id + future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id)) + + def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True): + metadata = NixlConnectorMetadata() + metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta( + local_block_ids=(blocks,), + local_physical_block_ids=(blocks,), + tp_size=1, + remote=RemoteMeta( + ([2],), "localhost", 1234, engine_id, f"{engine_id}-prefill" + ), + awaiting_kvs=awaiting_kvs, + ) + return metadata + + def test_cleans_old_peer_after_confirmed_handshake(self): + self._connect("healthy", host="other-host") + old_handle = self.worker.dst_xfer_side_handles["old"][0] + self.worker.start_load_kv(self._request()) + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_not_called() + + self._connect() + self.worker.get_transfer_results() + self.transport.release_dlist_handle.assert_called_once_with(old_handle) + self.transport.remove_remote_agent.assert_called_once_with("old") + assert set(self.worker._remote_agents) == {"new", "healthy"} + assert self.worker._engine_by_address == { + ("localhost", 1234): "new", + ("other-host", 1234): "healthy", + } + with pytest.raises(KeyError): + self.worker.transfer_topo.get_engine_info("old") + + self.worker.start_load_kv(NixlConnectorMetadata()) + assert self.worker.get_transfer_results().finished_recving == {"new-req"} + self.transport.remove_remote_agent.assert_called_once_with("old") + + @pytest.mark.parametrize("host,port", [("other-host", 1234), ("localhost", 5678)]) + def test_preserves_other_addresses(self, host, port): + self._connect(host=host, port=port) + self.worker.get_transfer_results() + assert "old" in self.worker._remote_agents + self.transport.remove_remote_agent.assert_not_called() + + def test_preserves_old_peer_when_handshake_fails(self): + self.worker.start_load_kv(self._request()) + self.worker._handshake_futures["new"].set_exception( + RuntimeError("handshake failed") + ) + assert self.worker.get_transfer_results().failed_recving == {"new-req"} + assert "old" in self.worker._remote_agents + assert self.worker._engine_by_address == {("localhost", 1234): "old"} + self.transport.remove_remote_agent.assert_not_called() + + @pytest.mark.parametrize( + "states,failed", + [ + (["PROC", "DONE"], False), + (["ERR", "PROC", "DONE"], True), + (["ERR", "PROC", "ERR"], True), + ], + ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"], + ) + def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed): + self.worker._recving_metadata.update(self._request("old").reqs_to_recv) + self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1] + self.transport.check_xfer_state.side_effect = states + self._connect() + + result = self.worker.get_transfer_results() + assert result.finished_recving == result.failed_recving == set() + assert self.worker.get_block_ids_with_load_errors() == set() + self.transport.remove_remote_agent.assert_not_called() + + result = self.worker.get_transfer_results() + assert result.finished_recving == {"old-req"} + assert result.failed_recving == ({"old-req"} if failed else set()) + assert self.worker.get_block_ids_with_load_errors() == ( + {1} if failed else set() + ) + assert self.transport.release_xfer_handle.call_count == (2 if failed else 1) + self.transport.remove_remote_agent.assert_called_once_with("old") + + def test_waits_for_queued_notification(self): + metadata = self._request("old", blocks=()) + self.worker._recving_metadata.update(metadata.reqs_to_recv) + self.worker._background_nixl_handshake( + "old-req", "old", metadata.reqs_to_recv["old-req"] + ) + self._connect() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_not_called() + + self.worker.start_load_kv(NixlConnectorMetadata()) + self.transport.send_notif.assert_called_once_with( + "old", notif_msg=b"old-prefill:1" + ) + result = self.worker.get_transfer_results() + assert result.finished_recving == {"old-req"} + assert result.failed_recving == set() + self.transport.remove_remote_agent.assert_called_once_with("old") + + def test_waits_for_other_handshakes(self): + self.worker._ensure_handshake("other", "other-host", 1234, 1) + self._connect() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_not_called() + + self._connect("other", host="other-host") + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_called_once_with("old") + + def test_rehandshakes_queued_request_for_released_engine(self): + # A request queued after its engine's handshake, but drained only after + # that engine was replaced and released, must not read from it. + metadata = self._request("old", blocks=(), awaiting_kvs=False) + self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"])) + self._connect() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_called_once_with("old") + + self.worker.start_load_kv(NixlConnectorMetadata()) + assert "old" in self.worker._handshake_futures + self.transport.send_notif.assert_not_called() + + def test_transfer_topology_unregister(): """TransferTopology.unregister_remote_engine removes the engine.""" from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 3be1730ec3..125dfa6143 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -4,6 +4,7 @@ import gc import queue +import threading from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -54,6 +55,9 @@ def region_pull_worker(): worker.enable_permute_local_kv = False worker.enable_heterogeneous_attn_post_process = False worker._engine_last_active = {} + worker._handshake_lock = threading.RLock() + worker._handshake_futures = {} + worker._transfer_layer_group_ids = () worker._bidirectional_kv_xfer_enabled = False worker._recving_transfers = {} worker.use_mla, worker._has_mamba = True, False @@ -94,6 +98,7 @@ def region_pull_worker(): worker.dst_num_blocks = {"P": 100, "D": 100} worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]} worker._remote_agents = {"P": {(0, 0): "P-rank0"}} + worker._engine_by_address = {("localhost", 1): "P"} worker._read_blocks_mixed = MagicMock() worker.nixl_wrapper = MagicMock() return worker diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index 231343de37..4d8ead28f3 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -395,6 +395,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): w._group_spec_types = (FullAttentionSpec,) w._engine_ttl = 0.0 w._engine_last_active = {} + w._engine_by_address = {} # Track _do_start_push_kv invocations. calls: list[tuple[str, Any, dict[str, Any]]] = [] diff --git a/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py new file mode 100644 index 0000000000..16358dec7e --- /dev/null +++ b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py @@ -0,0 +1,466 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for SimpleCPUOffloadConnector metrics.""" + +from __future__ import annotations + +import dataclasses +from types import SimpleNamespace +from typing import Any + +import pytest +from prometheus_client import Counter, Gauge, Histogram + +from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import ( + _make_connector, +) +from tests.v1.simple_kv_offload.test_scheduler import ( + make_request, + make_scheduler, + make_scheduler_output, +) +from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( + SimpleCPUOffloadConnector, +) +from vllm.v1.outputs import KVConnectorOutput +from vllm.v1.simple_kv_offload.manager import ( + BoundaryStoreStats, + LoadRequestState, + TransferMeta, +) +from vllm.v1.simple_kv_offload.metrics import ( + INFO_LABELS, + OUTCOME_TO_FIELD, + MetricName, + SimpleCPUOffloadStats, + _MetricType, + _StatsKey, +) + +SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES +LOAD_BLOCKS = MetricName.LOAD_BLOCKS +USED_BLOCKS = MetricName.USED_BLOCKS +PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS +INFO = MetricName.INFO + + +class _FakeMetric: + def __init__(self, **kwargs: Any): + self.kwargs = kwargs + self.children: list[_FakeMetric] = [] + self.observed: list[int | float] = [] + self.increments: list[int | float] = [] + self.set_values: list[int | float] = [] + self.labelvalues: tuple[object, ...] = () + + def labels(self, *labelvalues: object) -> _FakeMetric: + child = _FakeMetric(**self.kwargs) + child.labelvalues = labelvalues + self.children.append(child) + return child + + def observe(self, value: int | float) -> None: + self.observed.append(value) + + def inc(self, value: int | float) -> None: + self.increments.append(value) + + def set(self, value: int | float) -> None: + self.set_values.append(value) + + +def _fake_metric_types() -> dict[type, type]: + return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric} + + +def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any: + from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics + + return SimpleCPUOffloadPromMetrics( + vllm_config=SimpleNamespace(kv_transfer_config=None), + metric_types=_fake_metric_types(), + labelnames=["model_name", "engine"], + per_engine_labelvalues=per_engine or {0: ["model", "0"]}, + ) + + +def _stats_payload(types: dict, data: dict) -> dict[str, Any]: + return {_StatsKey.TYPES: types, _StatsKey.DATA: data} + + +def _values(stats: SimpleCPUOffloadStats, name: str) -> dict: + return stats.data[_StatsKey.DATA][name] + + +# --------------------------------------------------------------------------- +# Stats class +# --------------------------------------------------------------------------- + + +def test_build_kv_connector_stats_none_and_empty() -> None: + stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None) + assert isinstance(stats, SimpleCPUOffloadStats) + assert stats.is_empty() + + stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={}) + assert isinstance(stats, SimpleCPUOffloadStats) + assert stats.is_empty() + + +def test_stats_round_trip_through_serialized_dict() -> None: + stats = SimpleCPUOffloadStats() + stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats.increase_counter(LOAD_BLOCKS, 4) + stats.set_gauge(USED_BLOCKS, 7) + stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64")) + + rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict()) + assert isinstance(rebuilt, SimpleCPUOffloadStats) + assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3} + assert _values(rebuilt, LOAD_BLOCKS) == {(): 4} + assert _values(rebuilt, USED_BLOCKS) == {(): 7} + assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1} + + +def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None: + stats1 = SimpleCPUOffloadStats() + stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",)) + stats1.set_gauge(USED_BLOCKS, 7) + + stats2 = SimpleCPUOffloadStats() + stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",)) + stats2.set_gauge(USED_BLOCKS, 2) + + result = stats1.aggregate(stats2) + assert result is stats1 + outcomes = _values(stats1, SAVE_OUTCOMES) + assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1} + assert _values(stats1, USED_BLOCKS) == {(): 2} + + +def test_stats_aggregate_merges_types_from_other() -> None: + stats1 = SimpleCPUOffloadStats() + stats1.set_gauge(USED_BLOCKS, 1) + + stats2 = SimpleCPUOffloadStats() + stats2.increase_counter(LOAD_BLOCKS, 2) + + stats1.aggregate(stats2) + assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER + assert _values(stats1, LOAD_BLOCKS) == {(): 2} + + +def test_stats_reduce_and_reset() -> None: + stats = SimpleCPUOffloadStats() + stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats.set_gauge(PENDING_STORE_BLOCKS, 4) + stats.increase_counter(LOAD_BLOCKS, 9) + stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64")) + + reduced = stats.reduce() + assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3 + assert reduced[PENDING_STORE_BLOCKS] == 4 + assert reduced[LOAD_BLOCKS] == 9 + assert not any(key.startswith(INFO) for key in reduced) + + stats.reset() + assert stats.is_empty() + + +def test_outcome_labels_match_boundary_store_stats_fields() -> None: + """Every outcome label maps 1:1 to a BoundaryStoreStats counter field.""" + boundary_fields = { + f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published" + } + assert set(OUTCOME_TO_FIELD.values()) == boundary_fields + assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD) + + +# --------------------------------------------------------------------------- +# Prom metrics class +# --------------------------------------------------------------------------- + + +def test_prom_metrics_registers_tier1_metrics() -> None: + prom = _make_prom_metrics() + defs = prom._defs + assert set(defs) == { + SAVE_OUTCOMES, + LOAD_BLOCKS, + USED_BLOCKS, + PENDING_STORE_BLOCKS, + INFO, + } + # The prom class registers literal names (required by the docs + # generator); they must stay in sync with the emission-side constants. + assert {m.kwargs["name"] for m in defs.values()} == { + SAVE_OUTCOMES, + LOAD_BLOCKS, + USED_BLOCKS, + PENDING_STORE_BLOCKS, + INFO, + } + for metric in defs.values(): + assert metric.kwargs["documentation"] + assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"] + + assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [ + "model_name", + "engine", + "outcome", + ] + assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"] + assert defs[INFO].kwargs["labelnames"] == [ + "model_name", + "engine", + *INFO_LABELS, + ] + assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"] + + # mostrecent: multiprocess scrapes expose one series per label set, not + # one pid-tagged series per writer process. + for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO): + assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent" + + +def test_prom_metrics_observe_routes_each_series() -> None: + prom = _make_prom_metrics() + prom.observe( + _stats_payload( + types={ + SAVE_OUTCOMES: _MetricType.COUNTER, + LOAD_BLOCKS: _MetricType.COUNTER, + USED_BLOCKS: _MetricType.GAUGE, + PENDING_STORE_BLOCKS: _MetricType.GAUGE, + INFO: _MetricType.GAUGE, + }, + data={ + SAVE_OUTCOMES: {("stored",): 3}, + LOAD_BLOCKS: {(): 4}, + USED_BLOCKS: {(): 5}, + PENDING_STORE_BLOCKS: {(): 1}, + INFO: {("cpu", "false", "false", "64"): 1}, + }, + ) + ) + + assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3] + assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4] + assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5] + assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1] + assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1] + assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == ( + "model", + "0", + "stored", + ) + assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == ( + "model", + "0", + "cpu", + "false", + "false", + "64", + ) + + +def test_prom_metrics_reuses_bound_children() -> None: + prom = _make_prom_metrics() + payload = _stats_payload( + types={SAVE_OUTCOMES: _MetricType.COUNTER}, + data={SAVE_OUTCOMES: {("stored",): 1}}, + ) + prom.observe(payload) + prom.observe(payload) + + child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))] + assert child.increments == [1, 1] + assert len(prom._defs[SAVE_OUTCOMES].children) == 1 + + +def test_prom_metrics_rejects_unknown_metric() -> None: + prom = _make_prom_metrics() + with pytest.raises(AssertionError, match="Unknown"): + prom.observe( + _stats_payload( + types={"vllm:not_a_metric": _MetricType.COUNTER}, + data={"vllm:not_a_metric": {(): 1}}, + ) + ) + + +def test_prom_metrics_rejects_wrong_label_count() -> None: + prom = _make_prom_metrics() + with pytest.raises(AssertionError, match="labels"): + prom.observe( + _stats_payload( + types={SAVE_OUTCOMES: _MetricType.COUNTER}, + data={SAVE_OUTCOMES: {("stored", "extra"): 1}}, + ) + ) + + +def test_prom_metrics_routes_per_engine() -> None: + prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]}) + prom.observe( + _stats_payload( + types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}} + ), + engine_idx=1, + ) + assert (0, USED_BLOCKS, ()) not in prom._metrics + assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3] + + +def test_connector_build_prom_metrics() -> None: + from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics + + prom = SimpleCPUOffloadConnector.build_prom_metrics( + vllm_config=SimpleNamespace(kv_transfer_config=None), + metric_types=_fake_metric_types(), + labelnames=["model_name", "engine"], + per_engine_labelvalues={0: ["model", "0"]}, + ) + assert isinstance(prom, SimpleCPUOffloadPromMetrics) + + +# --------------------------------------------------------------------------- +# Manager: interval stats +# --------------------------------------------------------------------------- + + +def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None: + fixture = make_scheduler() + sched = fixture.scheduler + + sched.boundary_store_stats.stored += 3 + sched.boundary_store_stats.dropped_cpu_full += 2 + sched.boundary_store_stats.skipped_in_flight += 1 + + stats = sched.get_stats() + assert _values(stats, SAVE_OUTCOMES) == { + ("stored",): 3, + ("dropped_cpu_full",): 2, + ("skipped_in_flight",): 1, + } + + # Deltas are per-interval: a second drain reports no new outcomes. + stats2 = sched.get_stats() + assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA] + + +def test_manager_reset_preserves_undrained_outcome_deltas() -> None: + fixture = make_scheduler() + sched = fixture.scheduler + + sched.boundary_store_stats.stored += 3 + sched.reset() + + stats = sched.get_stats() + assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3} + + +def test_manager_get_stats_gauges_and_info_labels() -> None: + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + sched._store_event_to_blocks[0] = TransferMeta([10], [11]) + sched._pending_finished_stores.append(TransferMeta([12], [13])) + sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15]) + + stats = sched.get_stats() + assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3} + used = _values(stats, USED_BLOCKS)[()] + assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks() + + info = _values(stats, INFO) + assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1} + + # Gauges are re-reported every drain. + stats2 = sched.get_stats() + assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3} + + +def test_manager_info_labels_reflect_disk_mode() -> None: + connector = _make_connector( + extra_config={ + "kv_offload_backend": "disk", + "disk_path": "/tmp/fake-disk-metrics", + "disk_capacity_bytes": 1024**3, + "use_page_cache": True, + } + ) + sched = connector.scheduler_manager + assert sched is not None + assert sched._info_labelvalues[0] == "disk" + assert sched._info_labelvalues[1] == "true" + stats = sched.get_stats() + assert set(_values(stats, INFO)) == {sched._info_labelvalues} + + +def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None: + """CPU backend ignores use_page_cache; the info gauge must not claim it.""" + connector = _make_connector(extra_config={"use_page_cache": True}) + sched = connector.scheduler_manager + assert sched is not None + assert sched._info_labelvalues[:2] == ("cpu", "false") + stats = sched.get_stats() + assert set(_values(stats, INFO)) == {sched._info_labelvalues} + + +def test_manager_counts_load_blocks_completed() -> None: + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + request = make_request(num_blocks=2) + gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2) + cpu_blocks = sched.cpu_block_pool.get_new_blocks(2) + sched._reqs_to_load[request.request_id] = LoadRequestState( + request=request, + transfer_meta=TransferMeta( + [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks] + ), + ) + + sched.build_connector_meta(make_scheduler_output({})) + stats = sched.get_stats() + assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA] + + sched.update_connector_output( + KVConnectorOutput(finished_recving={request.request_id}) + ) + stats = sched.get_stats() + assert _values(stats, LOAD_BLOCKS) == {(): 2} + + +def test_manager_abandoned_load_counts_as_completed_after_reset() -> None: + """Loads abandoned by reset() that still finish on the worker are + counted as loaded blocks. + """ + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + request = make_request(num_blocks=2) + gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2) + cpu_blocks = sched.cpu_block_pool.get_new_blocks(2) + sched._reqs_to_load[request.request_id] = LoadRequestState( + request=request, + transfer_meta=TransferMeta( + [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks] + ), + ) + + sched.build_connector_meta(make_scheduler_output({})) + stats = sched.get_stats() + assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA] + + assert sched.reset() is False + assert set(sched._abandoned_reqs_to_load) == {request.request_id} + + sched.update_connector_output( + KVConnectorOutput(finished_recving={request.request_id}) + ) + assert sched._abandoned_reqs_to_load == {} + stats = sched.get_stats() + assert _values(stats, LOAD_BLOCKS) == {(): 2} diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index ed07718de6..3817a15e3e 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -2316,8 +2316,8 @@ def _make_hybrid_attention_mamba_scheduler( def test_hybrid_store_uses_resolved_group_block_sizes() -> None: """Replicated groups must not be scaled by the DCP world size. - ``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP - size and every other spec 1. Reconstructing a group's geometry as + The spec's ``dcp_sharded`` flag controls its effective block size. + Reconstructing a group's geometry as ``spec.block_size * cp_world_size`` over-scales the replicated groups, so the eager store scan believes far fewer of their blocks are ready and silently offloads only a fraction of them. diff --git a/tests/v1/spec_decode/test_dflash_prepare_inputs.py b/tests/v1/spec_decode/test_dflash_prepare_inputs.py index 16d6d8e516..1d004c3667 100644 --- a/tests/v1/spec_decode/test_dflash_prepare_inputs.py +++ b/tests/v1/spec_decode/test_dflash_prepare_inputs.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from types import SimpleNamespace +from unittest.mock import patch import numpy as np import pytest @@ -9,7 +10,7 @@ import torch from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.spec_decode.dflash.speculator import ( - prepare_dflash_inputs, + DFlashSpeculator, ) pytestmark = pytest.mark.skipif( @@ -24,6 +25,7 @@ def _run_prepare( cp_rank: int = 0, cp_size: int = 1, cp_interleave: int = 1, + draft_dcp_size: int | None = None, ): device = torch.device("cuda") max_num_reqs = 4 @@ -40,7 +42,9 @@ def _run_prepare( ) input_batch = SimpleNamespace( num_reqs=1, + num_tokens=4, num_scheduled_tokens=np.array([4], dtype=np.int32), + seq_lens_cpu_upper_bound=torch.tensor([target_positions[-1] + 1]), positions=torch.tensor(target_positions, dtype=torch.int64, device=device), query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device), idx_mapping=torch.tensor([2], dtype=torch.int32, device=device), @@ -74,36 +78,65 @@ def _run_prepare( next_prefill_tokens = torch.zeros_like(last_sampled) block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device) - prepare_dflash_inputs( - input_buffers, - query_slot_mapping, - context_positions, - context_slot_mapping, - sample_indices, - sample_pos, - sample_idx_mapping, - temperature, - seeds, - input_batch, - torch.tensor([1], dtype=torch.int32, device=device), - torch.tensor([2], dtype=torch.int32, device=device), - last_sampled, - next_prefill_tokens, - input_temperature, - input_seeds, - block_table, - 4, - cp_rank, - cp_size, - cp_interleave, - 123, - num_speculative_steps, - num_speculative_steps, - max_num_reqs, - max_num_tokens, - 128, + class InputsPrepared(Exception): + pass + + draft = SimpleNamespace( + input_buffers=input_buffers, + context_positions=context_positions, + _context_slot_mappings=context_slot_mapping.unsqueeze(0), + sample_indices=sample_indices, + sample_pos=sample_pos, + sample_idx_mapping=sample_idx_mapping, + temperature=temperature, + seeds=seeds, + hidden_states=torch.zeros(4, 1, device=device), + prepare_context_anchor=lambda *args: None, + query_cudagraph_manager=None, + dp_size=1, + dp_rank=0, + pcp_manager=None, + draft_kv_cache_group_id=0, + draft_kv_cache_group_ids=[0], + _layer_group_idx=None, + block_tables=SimpleNamespace( + slot_mappings=query_slot_mapping.unsqueeze(0), + input_block_tables=[block_table], + kernel_block_sizes=[4], + cp_rank=cp_rank, + cp_size=cp_size, + cp_interleave=cp_interleave, + ), + dcp_size=cp_size if draft_dcp_size is None else draft_dcp_size, + parallel_drafting_token_id=123, + num_query_per_req=num_speculative_steps, + num_speculative_steps=num_speculative_steps, + max_num_reqs=max_num_reqs, + max_num_tokens=max_num_tokens, + max_model_len=128, sample_from_anchor=True, ) + with ( + patch( + "vllm.v1.worker.gpu.spec_decode.dflash.speculator.dispatch_cg_and_sync_dp", + side_effect=InputsPrepared, + ), + pytest.raises(InputsPrepared), + ): + DFlashSpeculator.propose( + draft, + input_batch, + {}, + {}, + draft.hidden_states, + None, + torch.tensor([1], dtype=torch.int32, device=device), + torch.tensor([2], dtype=torch.int32, device=device), + last_sampled, + next_prefill_tokens, + input_temperature, + input_seeds, + ) torch.accelerator.synchronize() return SimpleNamespace( input_buffers=input_buffers, @@ -118,12 +151,17 @@ def _run_prepare( ) -def test_prepare_dflash_inputs_excludes_rejected_context_suffix(): +@pytest.mark.parametrize("cp_rank,cp_size", [(0, 1), (0, 2), (1, 2)]) +def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size): # Positions 10/11 use physical block 7. Rejected positions 12/13 would use # block 8, but must be PAD context rather than contaminating draft KV. out = _run_prepare( target_positions=[10, 11, 12, 13], block_table_values=[0, 0, 7, 8, 9, 10, 11, 12], + cp_rank=cp_rank, + cp_size=cp_size, + cp_interleave=2, + draft_dcp_size=1, ) assert out.context_positions[:4].tolist() == [10, 11, 0, 0] diff --git a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py index b542388952..2c4b951be7 100644 --- a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py +++ b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py @@ -17,6 +17,7 @@ from types import MethodType, SimpleNamespace from unittest.mock import patch import numpy as np +import pytest import torch from vllm.config.compilation import CUDAGraphMode @@ -53,6 +54,7 @@ def _make_fake_speculator( arange_np=np.arange(max_num_reqs + 1, dtype=np.int32), draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool), block_tables=fake_block_tables, + dcp_size=1, input_buffers=fake_input_buffers, attn_groups=[], kv_cache_config=SimpleNamespace(kv_cache_groups=[]), @@ -152,14 +154,17 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len(): assert torch.equal(bound, torch.tensor([1024, 503], dtype=torch.int32)) -def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(): +@pytest.mark.parametrize("draft_dcp_size", [1, 2]) +def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(draft_dcp_size): fake = _make_fake_speculator() + fake.dcp_size = draft_dcp_size fake.block_tables.cp_size = 2 fake.block_tables.cp_rank = 1 fake.block_tables.cp_interleave = 4 fake.input_buffers.seq_lens[:3] = torch.tensor([5, 9, 16]) def fake_prepare(out, seq_lens, num_reqs, dcp_size, dcp_rank, cp_interleave): + assert draft_dcp_size > 1 assert seq_lens is fake.input_buffers.seq_lens assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (3, 2, 1, 4) out[:num_reqs].copy_(torch.tensor([1, 4, 8], dtype=torch.int32)) @@ -176,6 +181,9 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(): ) local = captured["dcp_local_seq_lens"] + if draft_dcp_size == 1: + assert local is None + return assert isinstance(local, torch.Tensor) assert local.data_ptr() == fake.input_buffers.dcp_local_seq_lens.data_ptr() assert local.tolist() == [1, 4, 8, 0] diff --git a/tests/v1/worker/test_cp_utils.py b/tests/v1/worker/test_cp_utils.py index b38ae4d063..d477ace479 100644 --- a/tests/v1/worker/test_cp_utils.py +++ b/tests/v1/worker/test_cp_utils.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + import pytest import torch @@ -7,6 +9,38 @@ from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens from vllm.v1.worker.cp_utils import should_skip_dcp_context_attention +def test_cp_checks_allow_a_replicated_draft_backend_without_dcp(monkeypatch): + from vllm.v1.worker import cp_utils + + config = SimpleNamespace( + parallel_config=SimpleNamespace( + prefill_context_parallel_size=1, + decode_context_parallel_size=4, + cp_kv_cache_interleave_size=16, + ), + speculative_config=object(), + ) + target = SimpleNamespace( + impl=SimpleNamespace( + dcp_world_size=4, + supports_mtp_with_cp_non_trivial_interleave_size=True, + need_to_return_lse_for_decode=True, + ) + ) + draft = SimpleNamespace( + impl=SimpleNamespace( + dcp_world_size=1, + supports_mtp_with_cp_non_trivial_interleave_size=False, + need_to_return_lse_for_decode=False, + ) + ) + layers = {"target": target, "draft": draft} + monkeypatch.setattr(cp_utils, "get_layers_from_vllm_config", lambda *args: layers) + cp_utils.check_attention_cp_compatibility(config, target_layer_names={"target"}) + with pytest.raises(AssertionError, match="MTP with cp_kv_cache_interleave_size"): + cp_utils.check_attention_cp_compatibility(config) + + def test_skip_gate_only_for_zero_context(): assert should_skip_dcp_context_attention(torch.zeros(3, dtype=torch.int32)) assert not should_skip_dcp_context_attention( diff --git a/tests/v1/worker/test_dspark_utils.py b/tests/v1/worker/test_dspark_utils.py index 94655f192c..1014051d64 100644 --- a/tests/v1/worker/test_dspark_utils.py +++ b/tests/v1/worker/test_dspark_utils.py @@ -2,8 +2,17 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass, field +from types import SimpleNamespace +import pytest +import torch + +from vllm.config import ParallelConfig +from vllm.v1.kv_cache_interface import FullAttentionSpec +from vllm.v1.worker.gpu import model_runner +from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_config +from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator @dataclass @@ -49,3 +58,86 @@ def test_dspark_parallel_config_disables_eplb_atomically(): assert draft_config.eplb_config.num_redundant_experts == 0 assert not draft_config.enable_elastic_ep assert draft_config.eplb_config is not target_config.eplb_config + + +@pytest.mark.parametrize("pcp_size", [1, 4]) +@pytest.mark.parametrize("dcp_size", [1, 4]) +@pytest.mark.parametrize("use_mla", [False, True]) +def test_draft_context_parallelism_without_changing_target( + monkeypatch, pcp_size, dcp_size, use_mla +): + target_parallel = ParallelConfig( + tensor_parallel_size=4, + prefill_context_parallel_size=pcp_size, + decode_context_parallel_size=dcp_size, + cp_kv_cache_interleave_size=16, + distributed_executor_backend="mp", + ) + target_config = SimpleNamespace( + parallel_config=target_parallel, + speculative_config=SimpleNamespace( + draft_model_config=SimpleNamespace(use_mla=use_mla) + ), + ) + + class CapturedConfig(Exception): + pass + + def capture_init(self, config, device): + raise CapturedConfig(config) + + monkeypatch.setattr(DraftModelSpeculator, "__init__", capture_init) + with pytest.raises(CapturedConfig) as captured: + DFlashSpeculator(target_config, device=None) + draft_parallel = captured.value.args[0].parallel_config + assert draft_parallel.tensor_parallel_size == 4 + assert draft_parallel.prefill_context_parallel_size == 1 + assert draft_parallel.decode_context_parallel_size == (dcp_size if use_mla else 1) + assert draft_parallel.cp_kv_cache_interleave_size == 16 + assert target_parallel.prefill_context_parallel_size == pcp_size + assert target_parallel.decode_context_parallel_size == dcp_size + assert target_parallel.cp_kv_cache_interleave_size == 16 + + +@pytest.mark.parametrize("configured_dcp", [1, 4]) +def test_attention_uses_draft_dcp_setting_inside_target_process_group( + monkeypatch, configured_dcp +): + import vllm.config as config_module + from vllm.distributed import parallel_state + from vllm.v1.attention.backend import AttentionImplBase + + config = SimpleNamespace( + parallel_config=SimpleNamespace(decode_context_parallel_size=configured_dcp) + ) + monkeypatch.setattr( + config_module, "get_current_vllm_config_or_none", lambda: config + ) + + monkeypatch.setattr( + parallel_state, "_DCP", SimpleNamespace(world_size=4, rank_in_group=2) + ) + impl = AttentionImplBase() + assert impl.dcp_world_size == configured_dcp + assert impl.dcp_rank == (0 if configured_dcp == 1 else 2) + + +@pytest.mark.parametrize("draft_dcp_size", [1, 4]) +def test_runner_marks_only_replicated_draft_caches(monkeypatch, draft_dcp_size): + spec = FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=64, dtype=torch.bfloat16 + ) + monkeypatch.setattr( + model_runner, "get_kv_cache_spec", lambda _: {"target": spec, "draft": spec} + ) + runner = object.__new__(model_runner.GPUModelRunner) + runner.vllm_config = None + runner.dcp_size = 4 + runner.speculator = object.__new__(DFlashSpeculator) + runner.speculator.dcp_size = draft_dcp_size + runner.speculator.draft_attn_layer_names = {"draft"} + + specs = runner.get_kv_cache_spec() + + assert specs["target"].dcp_sharded + assert specs["draft"].dcp_sharded == (draft_dcp_size == 4) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index 4fc7d4d2e5..36ba9a4b16 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -88,6 +88,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode): query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32), seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32), ) + speculator.dcp_size = 1 speculator.block_tables = SimpleNamespace( cp_size=1, input_block_tables=[torch.zeros(4, 1, dtype=torch.int32)], @@ -183,6 +184,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected) parallel_config=SimpleNamespace( data_parallel_size=1, data_parallel_rank=0, + decode_context_parallel_size=1, ), ) diff --git a/tests/v1/worker/test_gpu_block_table.py b/tests/v1/worker/test_gpu_block_table.py index 5c245d1c4d..5c2231a1f2 100644 --- a/tests/v1/worker/test_gpu_block_table.py +++ b/tests/v1/worker/test_gpu_block_table.py @@ -171,22 +171,23 @@ def test_block_tables_skip_custom_slot_mapping_groups(): @pytest.mark.parametrize("cp_rank", range(4)) def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int): - """DCP interleave is expressed in logical-block token coordinates.""" + """Only sharded groups use DCP interleave in logical-block coordinates.""" device = torch.device("cuda") block_tables = BlockTables( - block_sizes=[128], + block_sizes=[128, 128], max_num_reqs=1, max_num_batched_tokens=1024, - max_num_blocks_per_group=[2], + max_num_blocks_per_group=[2, 8], device=device, - kernel_block_sizes=[64], + kernel_block_sizes=[64, 64], + dcp_sharded=[True, False], cp_size=4, cp_rank=cp_rank, cp_interleave=128, ) block_tables.append_block_ids( req_index=0, - new_block_ids=([5, 9],), + new_block_ids=([5, 9], list(range(10, 18))), overwrite=True, ) block_tables.apply_staged_writes() @@ -199,7 +200,7 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int): query_start_loc, positions, num_tokens_padded=1024, - )[0] + ) expected = torch.full((1024,), -1, dtype=torch.int64, device=device) first_start = cp_rank * 128 @@ -210,7 +211,8 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int): expected[second_start : second_start + 128] = torch.arange( 9 * 128, 10 * 128, dtype=torch.int64, device=device ) - assert torch.equal(actual, expected) + assert torch.equal(actual[0], expected) + assert torch.equal(actual[1], positions + 10 * 128) def test_v1_block_table_move_row_clears_vacated_row(): diff --git a/tests/v1/worker/test_gpu_input_batch.py b/tests/v1/worker/test_gpu_input_batch.py index 7c6c9ba417..f37e824e6c 100644 --- a/tests/v1/worker/test_gpu_input_batch.py +++ b/tests/v1/worker/test_gpu_input_batch.py @@ -372,6 +372,62 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis _compare_objs(input_batch, ref_input_batch) +@pytest.mark.parametrize("device", DEVICES) +def test_condense_clears_stale_allowed_token_ids_mask(device: str): + """condense() must clear the mask row a constrained request is moved out + of. Otherwise a later request that reuses that row without + allowed_token_ids inherits the stale whitelist. + See https://github.com/vllm-project/vllm/issues/43894. + """ + batch_size = 4 + allowed_token_id = 13 + input_batch = InputBatch( + max_num_reqs=batch_size, + max_model_len=1024, + max_num_batched_tokens=1024, + device=torch.device(device), + vocab_size=VOCAB_SIZE, + block_sizes=[1], + kernel_block_sizes=[1], + max_num_blocks_per_req=[1024], + ) + + def _make_req(suffix: int, allowed_token_ids=None) -> CachedRequestState: + return CachedRequestState( + req_id=f"req_id_{suffix}", + prompt_token_ids=[1, 2, 3], + sampling_params=SamplingParams(allowed_token_ids=allowed_token_ids), + pooling_params=None, + mm_features=[], + block_ids=([],), + generator=None, + num_computed_tokens=0, + output_token_ids=[], + ) + + # Only req_id_2 is constrained, and it lands on the highest row. + assert input_batch.add_request(_make_req(0)) == 0 + assert input_batch.add_request(_make_req(1)) == 1 + assert input_batch.add_request(_make_req(2, [allowed_token_id])) == 2 + + mask = input_batch.allowed_token_ids_mask_cpu_tensor + assert mask is not None + # The constrained row masks every token except the single allowed id. + assert not mask[2][allowed_token_id].item() + assert int(mask[2].sum().item()) == VOCAB_SIZE - 1 + + # Free row 0, then condense: req_id_2 slides from row 2 down into row 0. + input_batch.remove_request("req_id_0") + input_batch.condense() + assert input_batch.req_id_to_index["req_id_2"] == 0 + assert int(mask[2].sum().item()) == 0 + + # A new unrestricted request reuses row 2 and must stay unconstrained. + assert input_batch.add_request(_make_req(3)) == 2 + assert "req_id_3" not in input_batch.has_allowed_token_ids + assert int(mask[2].sum().item()) == 0 + + @pytest.mark.parametrize("device", DEVICES) def test_swap_states_preserves_allowed_token_ids_mask(device: str): """swap_states() must exchange the two allowed_token_ids_mask rows. diff --git a/tests/v1/worker/test_gpu_pcp_manager.py b/tests/v1/worker/test_gpu_pcp_manager.py index 0c4b5aff68..4612d49d92 100644 --- a/tests/v1/worker/test_gpu_pcp_manager.py +++ b/tests/v1/worker/test_gpu_pcp_manager.py @@ -75,6 +75,24 @@ def test_validate_config_rejects_full_graph_for_prefills(): ) +@pytest.mark.parametrize("draft_uses_mla", [False, True]) +@pytest.mark.parametrize("dcp_size", [1, 2, 4]) +def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size): + config = _make_config(CUDAGraphMode.NONE) + config.parallel_config.prefill_context_parallel_size = 4 + config.parallel_config.decode_context_parallel_size = dcp_size + config.model_config.hf_text_config.index_topk = 1 + config.speculative_config = SimpleNamespace( + use_dspark=lambda: True, + draft_model_config=SimpleNamespace(use_mla=draft_uses_mla), + ) + if draft_uses_mla and dcp_size == 2: + with pytest.raises(NotImplementedError, match="requires DCP=1 or DCP=PCP"): + PCPManager.validate_config(config, supports_mm_inputs=False) + return + PCPManager.validate_config(config, supports_mm_inputs=False) + + def test_replicated_decode_piecewise_graph_padding(monkeypatch): manager = PCPManager( pcp_world_size=2, diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index af0213361e..0e89b39818 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail( _OPS_REGISTERED = False +_AITER_SITUV2_ACT_ENV = { + "a8w4": "AITER_SITUV2_A8W4", + "a4w4": "AITER_SITUV2_A4W4", +} + + +def _resolve_situv2_activation() -> str: + """VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4. + + auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4. + """ + value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower() + if value in ("auto", "1"): + return "a4w4" + if value == "0": + return "a16w4" + return value + + def _sync_aiter_situv2_moe_env() -> None: - """Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env. + """Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2. - AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 / - AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first). - When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4 - (afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override. - - Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4 - dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs - afp4 FlyDSL but falls back to heuristic configs, which is not the - tuned a4w4 path this recipe is meant to use. + AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4, + so exactly one is set for a8w4/a4w4 and both are cleared for a16w4. """ import os - import vllm.envs as envs - - if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2: - os.environ["AITER_SITUV2_A4W4"] = "1" - os.environ.pop("AITER_SITUV2_A8W4", None) - else: - os.environ.pop("AITER_SITUV2_A4W4", None) + selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation()) + for name in _AITER_SITUV2_ACT_ENV.values(): + if name == selected: + os.environ[name] = "1" + else: + os.environ.pop(name, None) class rocm_aiter_ops: @@ -1917,7 +1928,8 @@ class rocm_aiter_ops: VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM. VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings. VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion. - VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4). + VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation + dtype (a16w4 | a8w4 | a4w4). VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM. Note: @@ -1987,7 +1999,7 @@ class rocm_aiter_ops: # TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE _TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE _MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS - _MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 + _MOE_SITUV2 = _resolve_situv2_activation() # TODO: Consolidate under _LINEAR_ENABLED _TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM # Lazily probed: whether aiter.topk_softmax supports the @@ -2016,7 +2028,7 @@ class rocm_aiter_ops: cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS - cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 + cls._MOE_SITUV2 = _resolve_situv2_activation() _sync_aiter_situv2_moe_env() cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY @@ -2132,9 +2144,25 @@ class rocm_aiter_ops: @classmethod @if_aiter_supported def is_fused_moe_situv2_enabled(cls) -> bool: - # _MOE_SITUV2 is a variant of aiter fused moe, so aiter - # fused moe must be enabled as well. - return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 + """True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected.""" + return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4" + + @classmethod + @if_aiter_supported + def get_fused_moe_situv2_activation(cls) -> str: + """SiTUv2 activation dtype name: a16w4, a8w4 or a4w4.""" + return cls._MOE_SITUV2 + + @classmethod + @if_aiter_supported + def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool: + """Whether the SiTUv2 kernels expect gate/up-interleaved w13. + + The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and + a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode + (experts/rocm_aiter_moe.py) must both use this. + """ + return cls._MOE_SITUV2 == "a8w4" @classmethod @if_aiter_supported diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 494f7f3938..d939fc5ad3 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1725,6 +1725,26 @@ class VllmConfig: ) self.compilation_config.mode = CompilationMode.NONE + # TODO: This is a stopgap that turns sequence parallelism / async TP + # off under batch invariance. Make the sequence-parallel reduce-scatter + # path batch-invariant instead so they can stay enabled (#56370). + # Sequence parallelism / async TP rewrite all_reduce + rms_norm into a + # reduce-scatter whose reduction order depends on the batch, so they + # are not batch-invariant. Decide this before the breakable-CUDA-graph + # default below, which declines to auto-enable when they are on. + pass_config = self.compilation_config.pass_config + if envs.VLLM_BATCH_INVARIANT and ( + pass_config.enable_sp or pass_config.fuse_gemm_comms + ): + logger.warning_once( + "Disabling sequence parallelism and async TP " + "(pass_config.enable_sp / fuse_gemm_comms) when " + "VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path " + "is not batch-invariant (see vllm-project/vllm#56370)." + ) + pass_config.enable_sp = False + pass_config.fuse_gemm_comms = False + breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph() if not breakable_cudagraph_enabled and ( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/base.py b/vllm/distributed/kv_transfer/kv_connector/v1/base.py index bc3c32ba4c..6df9d1cfbe 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/base.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/base.py @@ -42,6 +42,7 @@ The class provides the following primitives: import enum from abc import ABC, abstractmethod from collections.abc import Callable, Iterable +from contextlib import AbstractContextManager from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal @@ -227,6 +228,13 @@ class KVConnectorBase_V1(ABC): # Worker-side methods # ============================== + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Return a custom KV cache allocation context, if configured. + + Returning None uses the engine's default memory pool. + """ + return None + def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None: """Set the connector metadata from the scheduler. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py index a9255f7cda..cbb7339c91 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py @@ -59,7 +59,6 @@ from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.core.kv_cache_utils import ( - dcp_world_size_for_kv_cache_spec, resolve_dcp_kv_block_size, ) from vllm.v1.kv_cache_interface import ( @@ -259,13 +258,7 @@ class DecodeBenchConnectorScheduler: dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size self.kv_cache_groups = kv_cache_config.kv_cache_groups self.group_block_sizes = tuple( - resolve_dcp_kv_block_size( - group.kv_cache_spec, - dcp_world_size_for_kv_cache_spec( - group.kv_cache_spec, - dcp_world_size, - ), - ) + resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_world_size) for group in self.kv_cache_groups ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index eb4a7fafe9..dd64dd671e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -13,6 +13,7 @@ enabling prefix caching via hash-based deduplication. from collections import Counter from collections.abc import Iterable, Sequence +from contextlib import AbstractContextManager from typing import Any import torch @@ -322,6 +323,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): # Worker-side methods # ============================================================ + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Return a context manager for the custom MemPool, or None. + + Called by the Worker before ``initialize_kv_cache`` so that KV + cache is allocated from the Mooncake-managed pool when + ``custom_mem_pool`` is set in ``kv_connector_extra_config``. + """ + if self.connector_worker is None: + return None + return self.connector_worker.get_mem_pool_context() + def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): assert self.connector_worker is not None self.connector_worker.register_kv_caches(kv_caches) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index baf8967bc8..173aa324e7 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -19,13 +19,14 @@ import threading import time from collections.abc import Callable, Sequence from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import AbstractContextManager from typing import Any, TypeVar import torch import zmq import vllm.envs as envs -from vllm.config import VllmConfig +from vllm.config import ModelConfig, VllmConfig from vllm.distributed import ( get_dcp_group, get_pcp_group, @@ -1414,6 +1415,41 @@ class KVCacheStoreRecvingThread(KVTransferThread): class MooncakeStoreWorker: """Worker-side component for MooncakeStoreConnector.""" + @staticmethod + def _create_mem_pool( + extra_config: dict[str, Any], model_config: ModelConfig + ) -> torch.cuda.MemPool | None: + pool_type = str(extra_config.get("custom_mem_pool") or "").upper() + if not pool_type: + return None + if pool_type not in ("NVLINK", "BAREX"): + raise ValueError( + f"Unsupported custom_mem_pool={pool_type!r}, " + "expected 'NVLINK' or 'BAREX'" + ) + if model_config.enable_sleep_mode or model_config.enable_cumem_allocator: + raise ValueError( + "custom_mem_pool is incompatible with enable_sleep_mode " + "or enable_cumem_allocator; CuMemAllocator cannot manage " + "allocations from the custom pool." + ) + try: + if pool_type == "NVLINK": + from mooncake.allocator import NVLinkAllocator as allocator_cls + else: # pool_type == "BAREX" + from mooncake.allocator import BarexAllocator as allocator_cls + except ImportError as e: + raise ImportError( + f"custom_mem_pool={pool_type!r} requires " + "mooncake-transfer-engine>=0.3.8. Please upgrade Mooncake." + ) from e + + device = torch.device("cuda", torch.accelerator.current_device_index()) + allocator = allocator_cls.get_allocator(device) + mem_pool = torch.cuda.MemPool(allocator.allocator()) + logger.info("Using Mooncake custom memory pool: %s", pool_type) + return mem_pool + def __init__( self, vllm_config: VllmConfig, @@ -1471,6 +1507,10 @@ class MooncakeStoreWorker: # Initialize MooncakeDistributedStore with its own TransferEngine store_config = MooncakeStoreConfig.load_from_config() + # Validate and create the custom memory pool before initializing the + # store so failures cannot leak a partially initialized store handle. + self._mem_pool = self._create_mem_pool(extra_config, model_config) + self.store = MooncakeDistributedStore() local_ip = get_ip() local_hostname = rdma_utils.get_requester_local_hostname(local_ip) @@ -1741,6 +1781,13 @@ class MooncakeStoreWorker: ) return token_dbs + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Return a context manager for the custom MemPool, or None if + no custom pool is configured.""" + if self._mem_pool is None: + return None + return torch.cuda.use_mem_pool(self._mem_pool) + def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int: if self.dcp_size > 1: return 1 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index a6f0d4cafb..73b6b0ccf6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import copy from collections.abc import Callable, Iterable +from contextlib import AbstractContextManager from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast @@ -294,6 +295,26 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA): # ============================== # Worker-side methods # ============================== + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Forward a custom KV cache memory pool from a child connector. + + KV cache is allocated once and can only live in one pool, so at + most one child may provide a context. + """ + found: list[tuple[str, AbstractContextManager]] = [] + for connector in self._connectors: + if (ctx := connector.get_mem_pool_context()) is not None: + found.append((type(connector).__name__, ctx)) + + if len(found) > 1: + names = [name for name, _ in found] + raise ValueError( + f"Multiple connectors provide a KV cache memory pool {names}; " + "KV cache is allocated once and can only live in one pool. " + "Configure custom_mem_pool on at most one connector." + ) + return found[0][1] if found else None + def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: for c in self._connectors: c.start_load_kv(forward_context, **kwargs) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index b630dedfc3..f4a643d2b7 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -839,6 +839,8 @@ class NixlBaseConnectorWorker: self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config( "engine_ttl", 3600.0 ) + # Map of pull peer (host, port) -> engine_id currently serving it. + self._engine_by_address: dict[tuple[str, int], EngineId] = {} self.model_config = vllm_config.model_config @@ -1300,6 +1302,7 @@ class NixlBaseConnectorWorker: self._remote_agents[eid] = remote_agents self._engine_clock_offset[eid] = clock_offset self._engine_last_active[eid] = time.perf_counter() + self._track_remote_engine_replacement(eid, host, port) except Exception as e: self._log_failure( failure_type="handshake_setup_failed", @@ -2953,6 +2956,7 @@ class NixlBaseConnectorWorker: # Handle timeout to avoid stranding blocks on remote. self._reap_expired_send_leases(done_sending) + self._cleanup_replaced_remote_engines() return KVConnectorTransferResults( finished_sending=done_sending, finished_recving=done_recving, @@ -3541,14 +3545,53 @@ class NixlBaseConnectorWorker: and meta.remote is not None } + def _track_remote_engine_replacement( + self, engine_id: EngineId, host: str, port: int + ) -> None: + """Record the engine currently serving a pull peer address.""" + # TODO: Also handle push mode, which handshakes in both directions. + if self._TRANSFER_MODE != "pull": + return + previous = self._engine_by_address.get((host, port)) + self._engine_by_address[(host, port)] = engine_id + if previous is None or previous == engine_id: + return + logger.info( + "Remote engine at %s:%d changed from %s to %s.", + host, + port, + previous, + engine_id, + ) + + def _cleanup_replaced_remote_engines(self) -> None: + """Release replaced pull peers once requests and handshakes have drained.""" + if self._TRANSFER_MODE != "pull": + return + with self._handshake_lock: + if self._handshake_futures: + return + replaced = self._remote_agents.keys() - self._engine_by_address.values() + if not replaced: + return + # _recving_metadata is only accessed from this (main) thread. + busy = { + meta.remote.engine_id + for meta in self._recving_metadata.values() + if meta.remote is not None + } + for engine_id in replaced - busy: + self._cleanup_remote_engine(engine_id, log_eviction=False) + logger.info("Released NIXL state for replaced remote engine %s.", engine_id) + def _cleanup_remote_engine( self, engine_id: EngineId, *, log_eviction: bool = True ) -> None: """Remove all state for a single remote engine. Releases NIXL resources (dlist handles, remote agents) and clears - all per-engine data structures. Used by both TTL eviction and - shutdown. + all per-engine data structures. Used by TTL eviction, replaced peer + cleanup and shutdown. """ assert engine_id in self._remote_agents @@ -3558,6 +3601,9 @@ class NixlBaseConnectorWorker: # Pop under the handshake lock; NIXL teardown stays outside it. with self._handshake_lock: agents = self._remote_agents.pop(engine_id) + for address, eid in list(self._engine_by_address.items()): + if eid == engine_id: + del self._engine_by_address[address] for agent_name in agents.values(): self.nixl_wrapper.remove_remote_agent(agent_name) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index 6539b7e57e..11b7215950 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -84,7 +84,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): # Start transfers for requests whose handshakes have now finished. while not self._ready_requests.empty(): - self._read_blocks_for_req(*self._ready_requests.get_nowait()) + req_id, meta = self._ready_requests.get_nowait() + assert meta.remote is not None + if meta.remote.engine_id not in self._remote_agents: + # The engine was released after its handshake completed, so + # handshake again. This fails if the engine is gone. + self._background_nixl_handshake(req_id, meta.remote.engine_id, meta) + continue + self._read_blocks_for_req(req_id, meta) if self.pcp_rank > 0 and not self.pcp_dcp_sharded: # Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py index 64bf8146c9..9124d9eac1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py @@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorRole, SupportsHMA, ) +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) from vllm.logger import init_logger from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.outputs import KVConnectorOutput @@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import ( from vllm.v1.simple_kv_offload.metadata import ( SimpleCPUOffloadMetadata, ) +from vllm.v1.simple_kv_offload.metrics import ( + SimpleCPUOffloadPromMetrics, + SimpleCPUOffloadStats, +) from vllm.v1.simple_kv_offload.worker import ( SimpleCPUOffloadWorker, ) @@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): hash_block_size=hash_block_size, lazy_offload=lazy_offload, disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0, + use_page_cache=use_page_cache if disk_mode else False, ) elif role == KVConnectorRole.WORKER: self.worker_handler = SimpleCPUOffloadWorker( @@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): return self.scheduler_manager.get_boundary_store_stats() return None + def get_kv_connector_stats(self) -> KVConnectorStats | None: + # Worker-side callers may invoke this hook on connectors constructed + # via __new__ (as in test_worker.py), which lack scheduler_manager. + scheduler_manager: SimpleCPUOffloadScheduler | None = getattr( + self, "scheduler_manager", None + ) + if scheduler_manager is not None: + return scheduler_manager.get_stats() + return None + + @classmethod + def build_kv_connector_stats( + cls, data: dict[str, Any] | None = None + ) -> KVConnectorStats | None: + return ( + SimpleCPUOffloadStats(data=data) + if data is not None + else SimpleCPUOffloadStats() + ) + + @classmethod + def build_prom_metrics( + cls, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ) -> KVConnectorPromMetrics: + return SimpleCPUOffloadPromMetrics( + vllm_config, metric_types, labelnames, per_engine_labelvalues + ) + def take_events(self) -> Iterable[KVCacheEvent]: if self.scheduler_manager is not None: return self.scheduler_manager.take_events() diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index 20118649e1..88bb3d7c5a 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -1583,6 +1583,14 @@ def get_dcp_group() -> GroupCoordinator: return _DCP +def get_dcp_world_size_and_rank(enabled: bool = True) -> tuple[int, int]: + """Return ``(world_size, rank)`` in the DCP group, or ``(1, 0)`` when disabled + (e.g. a replicated draft cache) or the group is uninitialized (unit tests).""" + if not enabled or _DCP is None: + return 1, 0 + return _DCP.world_size, _DCP.rank_in_group + + _PP: GroupCoordinator | None = None diff --git a/vllm/envs.py b/vllm/envs.py index 78f99ea0ac..b0d72a71db 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -141,7 +141,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False VLLM_ROCM_USE_AITER_MOE: bool = True VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0 - VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False + VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto" VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto" @@ -1280,18 +1280,13 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_USE_AITER_MOE": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1") ), - # Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4 - # activations, separated gate/up layout) instead of default a16w4. vLLM - # sets AITER_SITUV2_A4W4 at init when this flag is on and clears any - # legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first). - # VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing - # recipes; it does not select a8w4 kernels. - # Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag - # and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics. - "VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: ( - os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1") - or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower() - in ("true", "1") + # Activation dtype for the Kimi-K3 SiTU MXFP4 MoE (AITER FlyDSL SiTUv2): + # auto (= a4w4), a4w4, a8w4 or a16w4. Legacy 1/0 mean a4w4/a16w4. + "VLLM_ROCM_USE_AITER_MOE_SITUV2": env_with_choices( + "VLLM_ROCM_USE_AITER_MOE_SITUV2", + "auto", + ["auto", "a4w4", "a8w4", "a16w4", "0", "1"], + case_sensitive=False, ), # MoE sorting dispatch policy for AITER fused MoE kernels. # 0 = auto (default): single-pass for small batches, multi-pass diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 62df395f69..4f2f760ad3 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -385,10 +385,11 @@ def rocm_aiter_fused_experts( gate_mode = "" if activation == MoEActivation.SITU: - # SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4 - # fp4 activations with separated gate/up weights (AITER #4463); - # default a16w4 SiTU also stays separated. - gate_mode = GateMode.SEPARATED.value + gate_mode = ( + GateMode.INTERLEAVE.value + if rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() + else GateMode.SEPARATED.value + ) elif quant_config.use_mxfp4_w4a16: gate_mode = GateMode.INTERLEAVE.value elif activation_interleave is not None: diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 5450f73f85..56ba3981b7 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -1596,8 +1596,7 @@ def convert_weight_to_mxfp4_moe_kernel_format( fp4_dtype = torch.float4_e2m1fn_x2 e8m0_dtype = torch.float8_e8m0fnu - # SiTUv2 flydsl uses separated gate/up layout (a4w4). - guinterleave = False + guinterleave = rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() w13 = rocm_aiter_ops.shuffle_weight_a16w4( w13_weight.data.view(fp4_dtype), 16, guinterleave ) diff --git a/vllm/models/deepseek_v41/attention.py b/vllm/models/deepseek_v41/attention.py index d0e69fb358..6d961f0055 100644 --- a/vllm/models/deepseek_v41/attention.py +++ b/vllm/models/deepseek_v41/attention.py @@ -697,7 +697,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): else: gemm_rs.warn_incompatible_projection() - def _wo_b_proj(self, z: torch.Tensor) -> torch.Tensor: + def _wo_b_proj(self, z: torch.Tensor | QuantizedActivation) -> torch.Tensor: """Apply ``wo_b``; with GEMM-RS bound, also reduce-scatter the result. Every ``_o_proj`` implementation projects through this so the decoder @@ -707,7 +707,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): """ if self.gemm_rs is None: return self.wo_b(z) - if self.gemm_rs.should_run(z): + if isinstance(z, torch.Tensor) and self.gemm_rs.should_run(z): return self.gemm_rs.apply(z, self.wo_b) # Small batches stay on the unfused path, which is faster there. return sp_reduce_scatter(self.wo_b(z)) diff --git a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py index 423849dcf1..0e02b486a5 100644 --- a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py @@ -9,15 +9,16 @@ import torch from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context -from vllm.models.deepseek_v4.nvidia.ops.o_proj import ( - compute_fp8_einsum_recipe, - deep_gemm_fp8_o_proj, -) +from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe from vllm.models.deepseek_v41.attention import DeepseekV4Attention from vllm.models.deepseek_v41.common.ops import ( build_flashinfer_mixed_sparse_indices, compute_global_topk_indices_and_lens, ) +from vllm.models.deepseek_v41.nvidia.ops.o_proj import ( + dsv41_o_proj, + register_dsv41_o_proj_warmup, +) from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4FlashMLAMetadata, DeepseekV4SparseMLABackend, @@ -242,27 +243,14 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): return _pad_to_supported_q_heads(num_heads) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe( self._o_proj_block_size ) + register_dsv41_o_proj_warmup(self) # Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the # per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``. if self.kv_cache_torch_dtype != torch.float8_e4m3fn: @@ -623,21 +611,7 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention): return _pad_to_supported_q_heads(num_heads) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None: super().__init__(vllm_config, *args, **kwargs) diff --git a/vllm/models/deepseek_v41/nvidia/flashmla.py b/vllm/models/deepseek_v41/nvidia/flashmla.py index 96dde36c61..ce3047ceca 100644 --- a/vllm/models/deepseek_v41/nvidia/flashmla.py +++ b/vllm/models/deepseek_v41/nvidia/flashmla.py @@ -6,16 +6,17 @@ from typing import TYPE_CHECKING, ClassVar, cast import torch from vllm.forward_context import get_forward_context -from vllm.models.deepseek_v4.nvidia.ops.o_proj import ( - compute_fp8_einsum_recipe, - deep_gemm_fp8_o_proj, -) +from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe from vllm.models.deepseek_v41.attention import DeepseekV4Attention from vllm.models.deepseek_v41.common.ops import ( combine_topk_swa_indices, compute_global_topk_indices_and_lens, dequantize_and_gather_k_cache, ) +from vllm.models.deepseek_v41.nvidia.ops.o_proj import ( + dsv41_o_proj, + register_dsv41_o_proj_warmup, +) from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, @@ -57,23 +58,10 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention): self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe( self._o_proj_block_size ) + register_dsv41_o_proj_warmup(self) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) @classmethod def get_padded_num_q_heads(cls, num_heads: int) -> int: diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py new file mode 100644 index 0000000000..b195d4b849 --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -0,0 +1,599 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DSV4.1 small-batch MXFP8 WO-A with inverse RoPE and quantization.""" + +from dataclasses import dataclass +from typing import Any + +import cutlass +import cutlass.cute as cute +import cutlass.utils as utils +import torch +from cuda.bindings.driver import CUstream +from cutlass import ( + BFloat16, + Float8E4M3FN, + Float32, + Int32, + Int64, + Uint8, + Uint32, + Uint64, +) +from cutlass._mlir.dialects import llvm +from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cute.runtime import make_fake_tensor +from cutlass.cutlass_dsl import dsl_user_op + +from vllm.cute_utils import _tcgen05, mbarrier, recast_val, simple_tma_copy +from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, kernel_launcher +from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import ( + CuTeDSLLaunchSpec, + VllmCuTeDSLJitKernel, +) + +# Each head is [nope | rope] as in fused_inv_rope_fp8_quant: only the last +# _ROPE_DIM columns carry RoPE. +_HEAD_DIM = 512 +_ROPE_DIM = 64 + + +@dsl_user_op +def _ue8m0_rp(value, *, loc=None, ip=None): + result = llvm.inline_asm( + Uint32.mlir_type, + [value.ir_value(loc=loc, ip=ip)], + "{ .reg .b16 sf; cvt.rp.satfinite.ue8m0x2.f32 sf, $1, $1; " + "cvt.u32.u16 $0, sf; and.b32 $0, $0, 255; }", + "=r,f", + has_side_effects=False, + loc=loc, + ip=ip, + ) + return Uint32(result) + + +@dsl_user_op +def _rope_fma(value, cos, other, sin, *, subtract=False, loc=None, ip=None): + """Preserve multiply/FMA rounding when RoPE loads are hoisted.""" + result = llvm.inline_asm( + Float32.mlir_type, + [v.ir_value(loc=loc, ip=ip) for v in (value, cos, other, sin)], + "{ .reg .f32 product; mul.rn.f32 product, $3, $4; " + + ( + "neg.f32 product, product; fma.rn.f32 $0, $1, $2, product; }" + if subtract + else "fma.rn.f32 $0, $1, $2, product; }" + ), + "=f,f,f,f,f", + has_side_effects=False, + loc=loc, + ip=ip, + ) + return Float32(result) + + +@cute.jit +def _scale(amax: Float32): + exponent = _ue8m0_rp(amax * Float32(1.0 / 448.0)) + inv = recast_val((Uint32(254) - exponent) << 23, Float32) + if exponent == 0: + inv = Float32(0) + return exponent, inv + + +class FusedWoAKernel(VllmCuTeDSLJitKernel["FusedWoAKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + # T=1..96 uses at most 46 keys per (n_groups, heads_per_group, o_lora_rank). + tokens_per_tile: int + token_tiles: int + full_tiles: bool + n_groups: int + heads_per_group: int + o_lora_rank: int + + @staticmethod + def kernel(compile_key: CompileKey) -> Any: + tokens = compile_key.tokens_per_tile + token_tiles = compile_key.token_tiles + full_tiles = compile_key.full_tiles + acc_cols = 1 << (tokens - 1).bit_length() + tile_m = max(8, acc_cols) + tile_n = 128 + tile_k = 128 + # tcgen05 MXFP8 MMA consumes K = 32, one MX block, per instruction. + mma_k = 32 + # Each group's heads split K across one cluster, one head per CTA. + heads_per_group = compile_key.heads_per_group + tiles_per_group = compile_key.o_lora_rank // tile_n + n_tiles = compile_key.n_groups * tiles_per_group + # Each CTA keeps one whole head of K in SMEM, one TMA box per stage. + num_stages = _HEAD_DIM // tile_k + w_stage_bytes = tile_n * tile_k + nope_dim = _HEAD_DIM - _ROPE_DIM + threads_per_token = _HEAD_DIM // 4 + sf_col = max(16, tile_m) + tmem_cols = sf_col * 2 + threads = min(acc_cols * threads_per_token, 512) + tokens_per_iter = threads // threads_per_token + + @cute.kernel + def device_kernel( + x: cute.Tensor, + positions: cute.Tensor, + rope: cute.Tensor, + w: cpasync.TmaInfo, + ws: cpasync.TmaInfo, + q: cute.Tensor, + qs: cute.Tensor, + num_tokens: Int32, + ): + total_tokens = num_tokens + if cutlass.const_expr(full_tiles): + total_tokens = Int32(tokens * token_tiles) + tid, _, _ = cute.arch.thread_idx() + bid, token_tile, _ = cute.arch.block_idx() + token_start = Int32(0) + if cutlass.const_expr(token_tiles > 1): + token_start = token_tile * tokens + warp = cute.arch.make_warp_uniform(tid // 32) + lane = tid % 32 + split = bid % heads_per_group + tile = bid // heads_per_group + group = tile // tiles_per_group + smem = utils.SmemAllocator() + sW = smem.allocate_tensor( + Float8E4M3FN, + w.smem_layout.outer, + byte_alignment=128, + swizzle=w.smem_layout.inner, + ) + sX = smem.allocate_tensor( + Float8E4M3FN, + cute.make_layout( + (tile_m, tile_k, num_stages), stride=(tile_k, 1, tile_m * tile_k) + ), + byte_alignment=1024, + swizzle=cute.make_swizzle(3, 4, 3), + ) + # tcgen05.cp 32x128b.warpx4 order: row r at byte (r % 32) * 16 + + # (r // 32) * 4, one 4-byte cell of MX-block scales per row. + sW_SF = smem.allocate_tensor( + Int32, + cute.make_layout(((32, 4), num_stages), stride=((4, 1), tile_n)), + 128, + ) + sW_SF_raw = smem.allocate_tensor( + Int32, cute.make_layout((tile_n, num_stages)), 128 + ) + sX_SF = smem.allocate_tensor( + Uint8, + cute.make_layout( + ((32, 4), tile_k // mma_k, num_stages), + stride=((16, 4), 1, tile_n * 4), + ), + 128, + ) + copy_fp8x4 = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), Float8E4M3FN, num_bits_per_copy=32 + ) + partial = smem.allocate_tensor( + Float32, + cute.make_layout( + (tile_n, cute.ceil_div(tokens, heads_per_group), heads_per_group) + ), + 128, + ) + mbar_loaded = smem.allocate_array(Int64, num_stages) + mbar_ws_loaded = smem.allocate_array(Int64, 1) + mbar_done = smem.allocate_array(Int64, 1) + mbar_reduced = smem.allocate_array(Int64, 1) + taddr = smem.allocate(Int32, 4) + if tid == 0: + for stage in cutlass.range_constexpr(num_stages): + # Wait for weight TMA and this stage's quantization threads. + cute.arch.mbarrier_init(mbar_loaded + stage, 1 + threads // 4) + cute.arch.mbarrier_init(mbar_ws_loaded, 1) + cute.arch.mbarrier_init(mbar_done, 1) + cute.arch.mbarrier_init(mbar_reduced, 1) + cute.arch.mbarrier_init_fence() + # Local users of the barriers (warp 0's TMA below) need the init too. + cute.arch.sync_threads() + # Paired with cluster_wait() below: peers must see mbar_reduced + # initialized before their st.async; the split hides the barrier. + cute.arch.cluster_arrive_relaxed() + if warp == 0: + # Raw DeepGEMM MN-major scales; the MMA warp transposes them to + # the UTCCP order, which TMA cannot scatter at 4-byte granularity. + ws_tiles = cute.zipped_divide(ws.tma_tensor, (tile_n, num_stages)) + with cute.arch.elect_one(): + mbarrier.arrive_expect_tx(mbar_ws_loaded, tile_n * num_stages * 4) + simple_tma_copy( + ws.atom, + ws_tiles[None, (tile % tiles_per_group, split, group)], + sW_SF_raw, + mbar_ws_loaded, + ) + tiles = cute.zipped_divide(w.tma_tensor, (tile_n, tile_k)) + for stage in cutlass.range_constexpr(num_stages): + with cute.arch.elect_one(): + mbarrier.arrive_expect_tx(mbar_loaded + stage, w_stage_bytes) + simple_tma_copy( + w.atom, + tiles[None, (tile, split * num_stages + stage)], + sW[None, None, stage], + mbar_loaded + stage, + ) + # x and positions come from the PDL predecessor; everything above + # only touches weights. Padded sX rows only feed unread MMA columns. + cute.arch.griddepcontrol_wait() + # TMEM is allocated at runtime, and a PDL predecessor on this SM may + # hold it until it exits; allocating earlier can deadlock. + if warp == 0: + cute.arch.alloc_tmem(tmem_cols, taddr) + cute.arch.relinquish_tmem_alloc_permit() + copy_x = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=64 + ) + n_iters = cute.ceil_div(tokens, tokens_per_iter) + qid = (tid + 32) % threads_per_token + k = qid * 4 + # Tail rows repeat the last input; their output stores are masked. + # Issue every token's x load before any math so their latencies overlap. + x_bf16 = cute.make_rmem_tensor((4, n_iters), BFloat16) + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: + x_src = cute.local_tile( + x[ + cute.min(token_start + token, total_tokens - 1), + group * heads_per_group + split, + None, + ], + (4,), + (qid,), + ) + cute.copy(copy_x, x_src, x_bf16[None, iteration]) + # Overlap the RoPE loads across tokens before consuming any of them. + cos_reg = cute.make_rmem_tensor((2, n_iters), Float32) + sin_reg = cute.make_rmem_tensor((2, n_iters), Float32) + if cutlass.const_expr(n_iters > 1): + copy_rope = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=32 + ) + if k >= nope_dim: + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if ( + cutlass.const_expr(tokens % tokens_per_iter == 0) + or token < tokens + ): + src_token = cute.min(token_start + token, total_tokens - 1) + pos = positions[src_token] + freq = (k - nope_dim) // 2 + rope_row = rope[pos, None] + cos_src = cute.local_tile(rope_row, (2,), (freq // 2,)) + cute.copy(copy_rope, cos_src, cos_reg[None, iteration]) + sin_tile = (freq + _ROPE_DIM // 2) // 2 + sin_src = cute.local_tile(rope_row, (2,), (sin_tile,)) + cute.copy(copy_rope, sin_src, sin_reg[None, iteration]) + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: + x_f32 = cute.make_rmem_tensor((4,), Float32) + x_f32.store(x_bf16[None, iteration].load().to(Float32)) + if k >= nope_dim: + # Inverse RoPE on the interleaved pairs (k, k+1) and + # (k+2, k+3); each rope row is cos || sin. + for pair in cutlass.range_constexpr(2): + if cutlass.const_expr(n_iters > 1): + cos = cos_reg[pair, iteration] + sin = sin_reg[pair, iteration] + else: + pos = positions[ + cute.min(token_start + token, total_tokens - 1) + ] + freq = (k - nope_dim) // 2 + pair + cos = Float32(rope[pos, freq]) + sin = Float32(rope[pos, freq + _ROPE_DIM // 2]) + even, odd = x_f32[2 * pair], x_f32[2 * pair + 1] + x_f32[2 * pair] = _rope_fma(even, cos, odd, sin) + x_f32[2 * pair + 1] = _rope_fma( + odd, cos, even, sin, subtract=True + ) + amax = cute.arch.fmax( + cute.arch.fmax(cute.abs(x_f32[0]), cute.abs(x_f32[1])), + cute.arch.fmax(cute.abs(x_f32[2]), cute.abs(x_f32[3])), + ) + amax = cute.arch.warp_reduction_max(amax, threads_in_group=8) + exponent, inv = _scale(cute.arch.fmax(amax, Float32(1e-10))) + x_fp8 = cute.make_rmem_tensor((4,), Float8E4M3FN) + x_fp8.store((x_f32.load() * inv).to(Float8E4M3FN)) + cute.copy( + copy_fp8x4, + x_fp8, + cute.local_tile( + sX[token, None, k // tile_k], (4,), (qid % 32,) + ), + ) + if lane % 8 == 0: + sX_SF[token, k % tile_k // mma_k, k // tile_k] = Uint8(exponent) + cute.arch.fence_proxy("async.shared", space="cta") + mbarrier.arrive(mbar_loaded + qid // 32, order="release") + if warp == 0: + base = cute.make_tensor(taddr, cute.make_layout(1))[0] + # sW / sX are 128B-swizzled K-major; one stage's K fills the + # swizzle atom, so the leading byte offset is unused. + sdesc = _tcgen05.make_sdesc_128B_swizzle(LBO=0) + # Unswizzled 32 x 16 B scale tiles: SBO = one 8-row core matrix + # (16 B units at bit 32); bit 46 is the SM100 descriptor version. + sf_sbo = 8 * 16 + sfdesc = Uint64((sf_sbo >> 4 << 32) | (1 << 46)) + # M = tile_n weight rows (A), N = tile_m token rows (B). + idesc = _tcgen05.make_mxfp8_idesc(tile_n, tile_m) + # sW_SF's layout is the UTCCP order, so a plain copy transposes; + # fence the generic-proxy writes before tcgen05.cp reads them. + cute.arch.mbarrier_wait(mbar_ws_loaded, 0) + for stage in cutlass.range_constexpr(num_stages): + for i in cutlass.range_constexpr(4): + sW_SF[i * 32 + lane, stage] = sW_SF_raw[i * 32 + lane, stage] + cute.arch.fence_proxy("async.shared", space="cta") + cute.arch.sync_warp() + for stage in cutlass.range_constexpr(num_stages): + cute.arch.mbarrier_wait(mbar_loaded + stage, 0) + _tcgen05.fence_after_thread_sync() + if cutlass.const_expr(stage == num_stages - 1): + with cute.arch.elect_one(): + cute.arch.griddepcontrol_launch_dependents() + adesc = sdesc | (sW[None, None, stage].iterator.toint() >> 4) + bdesc = sdesc | (sX[None, None, stage].iterator.toint() >> 4) + _tcgen05.cp( + base + sf_col, + sfdesc | (sW_SF[None, stage].iterator.toint() >> 4), + "32x128b", + "warpx4", + ) + _tcgen05.cp( + base + sf_col + 4, + sfdesc | (sX_SF[None, None, stage].iterator.toint() >> 4), + "32x128b", + "warpx4", + ) + # Advance mma_k FP8 bytes (16 B units) and pick MX block kk of + # each 4-byte scale (B sf_id at bit 4, A sf_id at bit 29). + for kk in cutlass.range_constexpr(tile_k // mma_k): + _tcgen05.mma_mxfp8( + base, + adesc + kk * mma_k // 16, + bdesc + kk * mma_k // 16, + idesc + ((kk << 4) | (kk << 29)), + base + sf_col, + base + sf_col + 4, + stage > 0 or kk > 0, + ) + _tcgen05.commit(mbar_done) + # Only the MMA warp waits; barrier 2 releases the TMEM readers. + cute.arch.mbarrier_wait(mbar_done, 0) + if tid < tile_n: + cute.arch.barrier(barrier_id=2, number_of_threads=tile_n) + # Tokens t with t % heads_per_group == split arrive from every peer + # (this CTA included), tile_n Float32 rows each. + if tid == 0: + owned = cute.ceil_div(tokens - split, heads_per_group) + mbarrier.arrive_expect_tx( + mbar_reduced, owned * tile_n * heads_per_group * 4 + ) + # Pairs with cluster_arrive_relaxed() above: every peer's + # mbar_reduced is initialized before the st.async below. + cute.arch.cluster_wait() + # quack's process-wide const_expr-if rewrite can leave these unbound + # after the dynamic path above; bind them so the regions join. + amax, exponent, inv = Float32(0), Uint32(0), Float32(0) + if tid < tile_n: + base = cute.make_tensor(taddr, cute.make_layout(1))[0] + _tcgen05.fence_after_thread_sync() + acc = cute.make_rmem_tensor(acc_cols, Float32) + if cutlass.const_expr(tokens == 1): + acc[0] = _tcgen05.ld(warp * 32, base, "32x32b", 1) + else: + acc.store(_tcgen05.ld(warp * 32, base, "32x32b", acc_cols)) + _tcgen05.wait_ld() + for token in cutlass.range_constexpr(tokens): + ptr = cute.domain_offset( + (tid, token // heads_per_group, split), partial + ).iterator + cute.arch.store_async_dsmem( + ptr, + recast_val(acc[token], Int32), + mbar_reduced, + token % heads_per_group, + ) + if split < tokens: + # One warp waits for the peers' bytes; named barrier 1 (0 is + # sync_threads) releases the other tile_n epilogue threads. + if warp == 0: + cute.arch.mbarrier_wait(mbar_reduced, 0) + cute.arch.barrier(barrier_id=1, number_of_threads=tile_n) + for i in cutlass.range_constexpr( + cute.ceil_div(tokens, heads_per_group) + ): + token = split + i * heads_per_group + if cutlass.const_expr( + full_tiles + and ( + tokens <= heads_per_group + or tokens % heads_per_group == 0 + ) + ) or (token < tokens and token_start + token < total_tokens): + value = Float32(0) + for peer in cutlass.range_constexpr(heads_per_group): + value += partial[tid, i, peer] + value = Float32(BFloat16(value)) + amax = cute.arch.warp_redux_sync(value, "max", abs=True) + exponent, inv = _scale(amax) + q[token_start + token, tile * tile_n + tid] = Float8E4M3FN( + value * inv + ) + if lane == 0: + row = token_start + token + offset = row % 32 * 16 + row // 32 * 4 + warp + qs[tile * 512 + offset] = Uint8(exponent) + # Every scale tile owns its padding, with no second memset kernel. + for i in cutlass.range_constexpr(4): + offset = i * tile_n + tid + row = offset // 16 + (offset % 16 // 4) * 32 + if token_tile == 0 and split == 0 and row >= total_tokens: + qs[tile * 512 + offset] = Uint8(0) + # Each receiver drained all writes into its own inbox. Only local + # TMEM readers must finish before deallocation; no peer reads SMEM. + if tid < tile_n: + cute.arch.barrier(barrier_id=2, number_of_threads=tile_n) + if warp == 0: + cute.arch.dealloc_tmem( + cute.make_ptr( + Float32, + cute.make_tensor(taddr, cute.make_layout(1))[0], + cute.AddressSpace.tmem, + ), + tmem_cols, + ) + + @cute.jit + def host_entrypoint( + x: cute.Tensor, + positions: cute.Tensor, + rope: cute.Tensor, + w: cute.Tensor, + ws: cute.Tensor, + q: cute.Tensor, + qs: cute.Tensor, + num_tokens: Int32, + stream: CUstream, + ): + layout = cute.make_composed_layout( + cute.make_swizzle(3, 4, 3), + 0, + cute.make_layout( + (tile_n, tile_k, num_stages), stride=(tile_k, 1, w_stage_bytes) + ), + ) + tma = cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE), + w, + layout, + (tile_n, tile_k), + ) + ws_tma = cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE), + cute.make_tensor(ws.iterator, cute.select(ws.layout, mode=[1, 2, 0])), + cute.make_layout((tile_n, num_stages)), + (tile_n, num_stages), + ) + device_kernel(x, positions, rope, tma, ws_tma, q, qs, num_tokens).launch( + grid=(n_tiles * heads_per_group, token_tiles, 1), + block=(threads, 1, 1), + cluster=(heads_per_group, 1, 1), + stream=stream, + use_pdl=True, + ) + + return host_entrypoint + + def dispatch( # type: ignore[override] + self, *, tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int + ) -> CompileKey: + token_tiles = (tokens + 31) // 32 + # Host integer arithmetic: balance tiles in multiples of four tokens. + tile_tokens = ( + tokens + if token_tiles == 1 + else (tokens + 4 * token_tiles - 1) // (4 * token_tiles) * 4 + ) + return self.CompileKey( + tokens_per_tile=tile_tokens, + token_tiles=token_tiles, + full_tiles=tokens % tile_tokens == 0, + n_groups=n_groups, + heads_per_group=heads_per_group, + o_lora_rank=o_lora_rank, + ) + + def get_warmup_keys( + self, *, max_tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int + ) -> list[CompileKey]: + # Trace all accepted token counts and deduplicate shared tile shapes. + return self._trace_dispatch(self.dispatch)( + tokens=WarmupIntRange(1, max_tokens + 1), + n_groups=n_groups, + heads_per_group=heads_per_group, + o_lora_rank=o_lora_rank, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: + tokens = ( + compile_key.tokens_per_tile * compile_key.token_tiles + if compile_key.full_tiles + else cute.sym_int() + ) + groups = compile_key.n_groups + rank = compile_key.o_lora_rank + n = groups * rank + k = compile_key.heads_per_group * _HEAD_DIM + return ( + make_fake_tensor( + BFloat16, + (tokens, groups * compile_key.heads_per_group, _HEAD_DIM), + (cute.sym_int(divisibility=8), _HEAD_DIM, 1), + assumed_align=16, + ), + make_fake_tensor(Int64, (tokens,), (1,)), + make_fake_tensor(Float32, (cute.sym_int(), _ROPE_DIM), (_ROPE_DIM, 1)), + make_fake_tensor(Float8E4M3FN, (n, k), (k, 1), assumed_align=16), + make_fake_tensor( + Int32, + (groups, rank, k // 128), + (rank * k // 128, 1, rank), + assumed_align=16, + ), + make_fake_tensor(Float8E4M3FN, (tokens, n), (n, 1)), + make_fake_tensor(Uint8, (128 * n // 32,), (1,)), + Int32(0), + ) + + @kernel_launcher + def __call__( + self, + *, + x: torch.Tensor, + positions: torch.Tensor, + rope: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + ) -> CuTeDSLLaunchSpec["FusedWoAKernel.CompileKey"]: + """Return FP8 WO-A output and FlashInfer F8_128x4 scale bytes.""" + tokens = x.shape[0] + groups, rank = weight_scale.shape[:2] + n = groups * rank + q = torch.empty((tokens, n), device=x.device, dtype=torch.float8_e4m3fn) + scales = torch.empty(128 * n // 32, device=x.device, dtype=torch.uint8) + launch_args = ( + x, + positions, + rope, + weight.view(n, -1), + weight_scale, + q, + scales, + tokens, + ) + compile_key = self.dispatch( + tokens=tokens, + n_groups=groups, + heads_per_group=x.shape[1] // groups, + o_lora_rank=rank, + ) + return compile_key, launch_args, (q, scales) + + +_FUSED_WO_A_KERNEL = FusedWoAKernel() diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py new file mode 100644 index 0000000000..4a315ddc9b --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DSV4.1 output projection with a small-batch SM100/SM103 fusion.""" + +import torch +from torch import nn + +from vllm.model_executor.kernels.linear.mxfp8.flashinfer import ( + FlashInferCutedslMxfp8LinearKernel, +) +from vllm.model_executor.layers.fusion.quant_activation import ( + QuantizedActivation, + get_input_quant_key, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic +from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj +from vllm.platforms import current_platform + + +def _fused_wo_a_max_tokens(layer: nn.Module) -> int: + max_tokens = getattr(layer, "_fused_wo_a_token_limit", None) + if max_tokens is None: + # Additional token tiles only pay off for smaller grids. + ctas_per_tile = layer.n_local_heads * (layer.o_lora_rank // 128) + max_tokens = 96 if ctas_per_tile <= 128 else 32 + layer._fused_wo_a_token_limit = max_tokens + return max_tokens + + +def _can_fuse_wo_a(layer: nn.Module) -> bool: + """Whether the attention layer matches the fused WO-A kernel's layout.""" + method = getattr(layer.wo_b, "scheme", layer.wo_b.quant_method) + return ( + current_platform.is_device_capability_family(100) + # WO-A is FP8 with per-32 scales, and WO-B takes MXFP8 input directly. + and layer._einsum_recipe == (1, 1, 32) + and get_input_quant_key(layer.wo_b) == kMxfp8Dynamic + and isinstance( + getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel + ) + # A group's heads form one portable (<= 8 CTA) cluster. + and layer.n_local_heads // layer.n_local_groups <= 8 + and (layer.nope_head_dim, layer.rope_head_dim) == (448, 64) + and layer.o_lora_rank % 128 == 0 + ) + + +def register_dsv41_o_proj_warmup(layer: nn.Module) -> None: + """Warm the fused WO-A token counts ``dsv41_o_proj`` can dispatch.""" + # Draft models load outside the warmup registry; their layers share the + # target's attention shapes, so the target's registration covers them too. + if _can_fuse_wo_a(layer): + from .fused_wo_a import _FUSED_WO_A_KERNEL + + _FUSED_WO_A_KERNEL.register_warmup( + max_tokens=_fused_wo_a_max_tokens(layer), + n_groups=layer.n_local_groups, + heads_per_group=layer.n_local_heads // layer.n_local_groups, + o_lora_rank=layer.o_lora_rank, + ) + + +def dsv41_o_proj( + layer: nn.Module, attn_out: torch.Tensor, positions: torch.Tensor +) -> torch.Tensor: + """``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100/SM103 batches.""" + o = attn_out[:, : layer.n_local_heads, :] + cos_sin_cache = layer.rotary_emb.cos_sin_cache + if 1 <= o.shape[0] <= _fused_wo_a_max_tokens(layer) and _can_fuse_wo_a(layer): + from .fused_wo_a import _FUSED_WO_A_KERNEL + + q, scales = _FUSED_WO_A_KERNEL( + x=o, + positions=positions, + rope=cos_sin_cache, + weight=layer.wo_a.weight, + weight_scale=layer.wo_a.weight_scale, + ) + return layer._wo_b_proj( + QuantizedActivation(q, scales, o.dtype, q.shape, kMxfp8Dynamic) + ) + return deep_gemm_fp8_o_proj( + o, + positions, + cos_sin_cache, + layer.wo_a, + layer._wo_b_proj, + n_groups=layer.n_local_groups, + heads_per_group=layer.n_local_heads // layer.n_local_groups, + nope_dim=layer.nope_head_dim, + rope_dim=layer.rope_head_dim, + o_lora_rank=layer.o_lora_rank, + einsum_recipe=layer._einsum_recipe, + tma_aligned_scales=layer._tma_aligned_scales, + ) diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py index 077dea91f0..9b28ed6dca 100644 --- a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py @@ -383,6 +383,7 @@ def _prefill_logits( page_table: torch.Tensor, query_start_loc: torch.Tensor, visible_blocks: torch.Tensor, + logits_workspace: torch.Tensor, max_query_len: int, logits_width: int, query_offset: int, @@ -392,10 +393,14 @@ def _prefill_logits( assert visible_blocks.shape == (q.shape[0],) assert 0 <= query_offset <= query_offset + num_queries <= q.shape[0] assert 0 < logits_width <= page_table.shape[1] * k_cache.shape[1] + assert logits_workspace.is_contiguous() + assert logits_workspace.numel() >= num_queries * logits_width - logits = torch.empty( - (num_queries, logits_width), dtype=torch.float32, device=q.device + # slice from pre-allocated workspace + logits = logits_workspace[: num_queries * logits_width].view( + num_queries, logits_width ) + # tuned on GB300 if k_cache.dtype == torch.float8_e4m3fn: TILE_R, STAGES, num_warps = 32, 2, 8 @@ -610,8 +615,13 @@ def qsa_select_paged_prefill( logits_width = min(max(64, logits_width), page_table.shape[1] * k_cache.shape[1]) # chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB + # always allocate the worst case to avoid memory fragmentation. max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024 rows_per_chunk = max(1, max_logits_bytes // (logits_width * 4)) + + budget_bytes = max(max_logits_bytes, page_table.shape[1] * k_cache.shape[1] * 4) + logits_workspace = q.new_empty(budget_bytes // 4, dtype=torch.float32) + topk_workspace = torch.empty( (_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device ) @@ -625,6 +635,7 @@ def qsa_select_paged_prefill( page_table, query_start_loc, visible_blocks, + logits_workspace, max_query_len, logits_width, query_offset=query_start, diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index b5cd6e4b32..5d661634d7 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -870,15 +870,14 @@ class AttentionImplBase(ABC, Generic[T]): def __new__(cls, *args, **kwargs): # use __new__ so that all subclasses will call this self = super().__new__(cls) - try: - from vllm.distributed.parallel_state import get_dcp_group + from vllm.config import get_current_vllm_config_or_none + from vllm.distributed.parallel_state import get_dcp_world_size_and_rank - self.dcp_world_size = get_dcp_group().world_size - self.dcp_rank = get_dcp_group().rank_in_group - except AssertionError: - # DCP might not be initialized in testing - self.dcp_world_size = 1 - self.dcp_rank = 0 + # Replicated drafts run at DCP=1 inside a DCP target's process group. + config = get_current_vllm_config_or_none() + self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank( + config is None or config.parallel_config.decode_context_parallel_size > 1 + ) try: from vllm.distributed.parallel_state import get_pcp_group diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 129b3e39c3..6035678394 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -60,7 +60,10 @@ from vllm.config import ( get_layers_from_vllm_config, ) from vllm.config.cache import CacheDType -from vllm.distributed.parallel_state import get_dcp_group +from vllm.distributed.parallel_state import ( + get_dcp_group, + get_dcp_world_size_and_rank, +) from vllm.logger import init_logger from vllm.platforms.interface import DeviceCapability from vllm.utils.math_utils import cdiv, round_up @@ -726,15 +729,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad num_qo_heads=self.num_heads_q, ) - try: - from vllm.distributed.parallel_state import get_dcp_group - - self.dcp_world_size = get_dcp_group().world_size - self.dcp_rank = get_dcp_group().rank_in_group - except AssertionError: - # DCP might not be initialized in testing - self.dcp_world_size = 1 - self.dcp_rank = 0 + self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank( + kv_cache_spec.dcp_sharded + ) # Fused draft decode reuses the captured metadata object across draft # steps. For DCP, build-time host-side decisions such as diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index f40c75b474..b3d40b1c7f 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -30,7 +30,10 @@ from vllm.config import ( get_current_vllm_config_or_none, ) from vllm.config.cache import CacheDType -from vllm.distributed.parallel_state import get_dcp_group +from vllm.distributed.parallel_state import ( + get_dcp_group, + get_dcp_world_size_and_rank, +) from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, @@ -641,6 +644,7 @@ class FlashInferTrtllmAPIDecode: @dataclass class FlashInferMetadata: + kv_cache_layout: KVCacheLayout num_actual_tokens: int """Total number of tokens in the batch (excluding padding).""" @@ -747,17 +751,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self._decode_cudagraph_max_bs, self.compilation_config.max_cudagraph_capture_size, ) - try: - self.dcp_world_size = get_dcp_group().world_size - self.dcp_rank = get_dcp_group().rank_in_group - self.dcp_kv_cache_interleave_size = ( - vllm_config.parallel_config.cp_kv_cache_interleave_size - ) - except AssertionError: - # DCP might not be initialized in testing - self.dcp_world_size = 1 - self.dcp_rank = 0 - self.dcp_kv_cache_interleave_size = 1 + self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank( + kv_cache_spec.dcp_sharded + ) + self.dcp_kv_cache_interleave_size = ( + vllm_config.parallel_config.cp_kv_cache_interleave_size + ) self.use_dcp = self.dcp_world_size > 1 self.dcp_a2a = ( self.use_dcp and vllm_config.parallel_config.dcp_comm_backend == "a2a" @@ -1424,6 +1423,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # Leave prefill/decode/cascade_wrapper empty, to be populated # case by case depending on the batch contents and backend selection. attn_metadata = FlashInferMetadata( + kv_cache_layout=self.kv_cache_layout, num_actual_tokens=num_actual_tokens, slot_mapping=common_attn_metadata.slot_mapping, q_data_type_prefill=self.q_data_type_prefill, @@ -1904,8 +1904,6 @@ class FlashInferImpl(AttentionImpl): num_heads, num_kv_heads, is_prefill=False ) vllm_config = get_current_vllm_config_or_none() - # The layout is resolved after model construction, so read it lazily. - self.cache_config = vllm_config.cache_config if vllm_config else None # Query pre-quantization needs a single dtype for the whole query tensor. # SM90 XQA needs BF16/FP16-Q for decode and FP8 for prefill, # so only enable this for SM100 trtllm-gen where both use FP8-Q. @@ -1941,11 +1939,6 @@ class FlashInferImpl(AttentionImpl): else: self.dcp_combine = partial(cp_lse_ag_out_rs, is_lse_base_on_e=False) - @property - def kv_cache_layout(self) -> KVCacheLayout: - assert self.cache_config is not None - return self.cache_config.get_resolved_kv_cache_layout() - def fused_output_quant_supported(self, quant_key: QuantKey): if quant_key == kNvfp4Dynamic and self.is_kvcache_nvfp4: logger.warning_once( @@ -2111,6 +2104,7 @@ class FlashInferImpl(AttentionImpl): # performance to make sure it does not introduce any overhead. num_actual_tokens = attn_metadata.num_actual_tokens + kv_cache_layout = attn_metadata.kv_cache_layout # FlashInfer treats uint8 KV cache as NVFP4. vLLM stores FP8 KV cache # as uint8 bytes, so pass FP8 caches with their logical dtype. @@ -2134,7 +2128,7 @@ class FlashInferImpl(AttentionImpl): if attn_metadata.use_cascade: # Cascade attention (rare case). assert attn_metadata.cascade_wrapper is not None - stride_order = self.kv_cache_layout.layer_view_order + stride_order = kv_cache_layout.layer_view_order if self.is_kvcache_nvfp4: kv_cache_views = tuple( cache.permute(*stride_order) @@ -2154,7 +2148,7 @@ class FlashInferImpl(AttentionImpl): num_decode_tokens = attn_metadata.num_decode_tokens num_prefill_tokens = attn_metadata.num_prefill_tokens - stride_order = self.kv_cache_layout.layer_view_order + stride_order = kv_cache_layout.layer_view_order kv_cache_permute = kv_cache.permute(*stride_order) # HND and contiguous # Fix degenerate strides on any size-1 dimension (e.g. num_kv_heads=1 # with TP=8). PyTorch permits non-canonical strides on size-1 dims; @@ -2311,7 +2305,7 @@ class FlashInferImpl(AttentionImpl): seq_lens_prefill = attn_metadata.prefill.seq_lens # This path needs to be enabled with VLLM_KV_CACHE_LAYOUT = HND - assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND" + assert get_flashinfer_layout_string(kv_cache_layout) == "HND" assert is_strictly_contiguous(prefill_query) assert is_strictly_contiguous(workspace_buffer) assert is_strictly_contiguous(block_tables_prefill) @@ -2501,7 +2495,7 @@ class FlashInferImpl(AttentionImpl): # trtllm-gen needs HND layout on SM100. XQA is selected # separately on SM90 and does not use this SM100 layout gate. if decode_with_trtllm_gen: - assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND" + assert get_flashinfer_layout_string(kv_cache_layout) == "HND" else: assert decode_with_xqa assert is_strictly_contiguous(decode_query) @@ -2547,7 +2541,7 @@ class FlashInferImpl(AttentionImpl): window_left=self.window_left, out=output_padded[:decode_query_tokens], sinks=self.sinks, - kv_layout=get_flashinfer_layout_string(self.kv_cache_layout), + kv_layout=get_flashinfer_layout_string(kv_cache_layout), q_len_per_req=q_len_per_req, mask=attn_metadata.decode.mask, q_cu_seq_lens=attn_metadata.decode.q_cu_seq_lens, @@ -2614,7 +2608,7 @@ class FlashInferImpl(AttentionImpl): sinks=self.sinks, o_sf_scale=self.o_sf_scale, out=out, - kv_layout=get_flashinfer_layout_string(self.kv_cache_layout), + kv_layout=get_flashinfer_layout_string(kv_cache_layout), backend=attn_metadata.decode.kernel.value, q_len_per_req=q_len_per_req, max_q_len=max_q_len, diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index e918774088..c97483912d 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -21,7 +21,12 @@ from vllm.config.cache import _layout_from_name from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor -from vllm.v1.kv_cache_interface import KVCacheLayout, KVCacheSpec, MambaSpec +from vllm.v1.kv_cache_interface import ( + AttentionSpec, + KVCacheLayout, + KVCacheSpec, + MambaSpec, +) if TYPE_CHECKING: from vllm.v1.core.sched.output import SchedulerOutput @@ -286,9 +291,10 @@ def resolve_kv_cache_layout( # specs can re-interpret HNC with different sizes as long as the total number of # bytes is the same. If not block-compact, each spec must agree on HNC to alias # the same page (this aliasing is done by the Hybrid Memory Allocator, HMA). + kv_cache_specs = tuple(kv_cache_specs or ()) hnc_shapes = { (spec.num_heads, spec.num_states, spec.page_size_bytes) - for spec in kv_cache_specs or () + for spec in kv_cache_specs } if len(hnc_shapes) > 1: candidates = [m for m in candidates if m.is_block_compact] @@ -298,6 +304,20 @@ def resolve_kv_cache_layout( f"none is in every supported set: {supported_layouts}." ) + dcp_sharding = { + spec.dcp_sharded for spec in kv_cache_specs if isinstance(spec, AttentionSpec) + } + page_sizes = {spec.page_size_bytes for spec in kv_cache_specs} + if len(dcp_sharding) > 1 and len(page_sizes) > 1: + # Sharded target and replicated draft caches need independent groups. + # Block-outer layouts can pack those groups without equalizing pages. + candidates = [layout for layout in candidates if layout.is_block_outermost] + if not candidates: + raise ValueError( + "DCP with a replicated draft and mixed KV page sizes requires " + f"a block-outer KV cache layout; supported sets: {supported_layouts}." + ) + if (requested := envs.VLLM_KV_CACHE_LAYOUT) is not None: layout = _layout_from_name(requested) if layout not in candidates: diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index 8061999409..d69b2f5fb3 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -11,7 +11,6 @@ from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector from vllm.v1.core.kv_cache_utils import ( BlockHash, KVCacheBlock, - dcp_world_size_for_kv_cache_spec, ) from vllm.v1.core.single_type_kv_cache_manager import ( CrossAttentionManager, @@ -20,6 +19,7 @@ from vllm.v1.core.single_type_kv_cache_manager import ( get_manager_for_kv_cache_spec, ) from vllm.v1.kv_cache_interface import ( + AttentionSpec, FullAttentionSpec, KVCacheConfig, KVCacheSpec, @@ -140,9 +140,7 @@ class KVCacheCoordinator(ABC): role=kv_cache_group.role, enable_caching=enable_caching, kv_cache_group_id=i, - dcp_world_size=dcp_world_size_for_kv_cache_spec( - kv_cache_group.kv_cache_spec, dcp_world_size - ), + dcp_world_size=dcp_world_size, pcp_world_size=pcp_world_size, scheduler_block_size=self.scheduler_block_size, needs_kv_cache_zeroing=self.kv_cache_config.needs_kv_cache_zeroing, @@ -664,16 +662,18 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): f"hash_block_size. block_sizes={cacheable_block_sizes}, " f"hash_block_size={hash_block_size}" ) - assert pcp_world_size == 1, "PCP not support hybrid attn now." - if dcp_world_size > 1: - # DCP shards full-attention KV across ranks and replicates Mamba - # state; other spec types (e.g. sliding window) have no DCP-aware - # handling yet, so reject them explicitly. - for g in kv_cache_config.kv_cache_groups: - assert isinstance(g.kv_cache_spec, (FullAttentionSpec, MambaSpec)), ( + for g in kv_cache_config.kv_cache_groups: + spec = g.kv_cache_spec + replicated = not spec.dcp_sharded + if pcp_world_size > 1: + assert isinstance(spec, FullAttentionSpec) or ( + isinstance(spec, AttentionSpec) and replicated + ), "PCP only supports full attention and replicated draft groups." + if dcp_world_size > 1: + assert isinstance(spec, FullAttentionSpec) or replicated, ( "DCP with hybrid KV cache layouts only supports " - "full-attention and Mamba groups, got: " - f"{type(g.kv_cache_spec).__name__}." + "full-attention, Mamba, and replicated draft groups, got: " + f"{type(spec).__name__}." ) # Fine-grained hash hits require Mamba "align" and compatible cache # managers in every group. TP needs hashing finer than the Mamba block; diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index ee0957839e..3f4f4e7bb1 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -682,12 +682,7 @@ def hash_block_tokens( def resolve_dcp_kv_block_size(spec: KVCacheSpec, dcp_world_size: int) -> int: """Return the token span of a cache block under DCP.""" - layer_specs = iter_layer_specs(spec) - if len(layer_specs) > 0 and all( - isinstance(layer_spec, AttentionSpec) for layer_spec in layer_specs - ): - return spec.block_size * dcp_world_size - return spec.block_size + return spec.block_size * (dcp_world_size if spec.dcp_sharded else 1) def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCacheSpec: @@ -707,28 +702,6 @@ def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCache return replace(spec, block_size=block_size) -def dcp_world_size_for_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> int: - """Return the DCP size that owns this group's block geometry. - - Full-attention KV (including MLA) is sharded across DCP ranks, so prefix - hashing and manager ``block_size`` use the process DCP size. Other specs - keep replicated per-rank state (Mamba, sliding window, chunked-local) and - must keep ``dcp_world_size=1`` even when the process runs with DCP > 1. - - Draft MLA groups on the sharded DSpark path are ``FullAttentionSpec`` / - ``MLAAttentionSpec`` and therefore keep the process DCP size. A replicated - draft group would need a different spec, not this helper. - """ - if dcp_world_size <= 1: - return 1 - inner = spec - if isinstance(spec, UniformTypeKVCacheSpecs): - inner = next(iter(spec.kv_cache_specs.values())) - if isinstance(inner, FullAttentionSpec): - return dcp_world_size - return 1 - - def resolve_kv_cache_block_sizes( kv_cache_config: KVCacheConfig, vllm_config: VllmConfig, @@ -752,6 +725,8 @@ def resolve_kv_cache_block_sizes( groups = kv_cache_config.kv_cache_groups if len(groups) <= 1: + if groups and not groups[0].kv_cache_spec.dcp_sharded: + dcp = 1 bs = cache_config.block_size * dcp return bs, bs @@ -1935,18 +1910,21 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]): kv_cache_spec: The kv cache spec of each attention layer in the model """ - if is_kv_cache_spec_uniform( - kv_cache_spec - ) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec): - return - - logger.warning( - "Hybrid KV cache manager is disabled for this hybrid model, " - "This means we do not enable any optimizations for saving KV cache " - "memory (e.g., dropping the KV cache outside the sliding window). " - "The compute of layers like sliding window is still saved." - ) - kv_cache_spec.update(_promote_local_kv_cache_specs(kv_cache_spec)) + groups: defaultdict[bool, dict[str, KVCacheSpec]] = defaultdict(dict) + for name, spec in kv_cache_spec.items(): + replicated = isinstance(spec, AttentionSpec) and not spec.dcp_sharded + groups[replicated][name] = spec + for specs in groups.values(): + promoted_specs = _promote_local_kv_cache_specs(specs) + if promoted_specs == specs: + continue + logger.warning( + "Hybrid KV cache manager is disabled for this hybrid model, " + "This means we do not enable any optimizations for saving KV cache " + "memory (e.g., dropping the KV cache outside the sliding window). " + "The compute of layers like sliding window is still saved." + ) + kv_cache_spec.update(promoted_specs) def _approximate_gcd(values: Sequence[int], *, lower_bound: int | None = None) -> int: @@ -2276,7 +2254,7 @@ def _ensure_min_page_size( scaled: list[KVCacheGroupSpec] = [] for g in groups: s = g.kv_cache_spec - kw: dict[str, int] = {"block_size": s.block_size * scale} + kw: dict[str, Any] = {"block_size": s.block_size * scale} if isinstance(s, (AttentionSpec, MambaSpec)) and s.page_size_padded is not None: kw["page_size_padded"] = s.page_size_padded * scale scaled.append(KVCacheGroupSpec(g.layer_names, replace(s, **kw))) diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 0dd9734847..7352f71cb6 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -106,11 +106,9 @@ class SingleTypeKVCacheManager(ABC): # managers have been validated by the coordinator. self.cache_hit_alignment_tokens = scheduler_block_size # The block size for this manager; used for actual block allocation. - self.block_size = kv_cache_spec.block_size - self.dcp_world_size = dcp_world_size - self.pcp_world_size = pcp_world_size - if dcp_world_size > 1: - self.block_size *= dcp_world_size + self.dcp_world_size = dcp_world_size if kv_cache_spec.dcp_sharded else 1 + self.pcp_world_size = pcp_world_size if kv_cache_spec.dcp_sharded else 1 + self.block_size = kv_cache_spec.block_size * self.dcp_world_size self.kv_cache_spec = kv_cache_spec self.block_pool = block_pool self.enable_caching = enable_caching @@ -1455,10 +1453,6 @@ class MambaManager(SingleTypeKVCacheManager): self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs ) -> None: super().__init__(kv_cache_spec, block_pool, **kwargs) - # Mamba layers use TP instead of DCP, so each rank holds the full - # recurrent state. Undo the DCP/PCP block_size scaling that the base - # class applies for attention groups whose KV cache is partitioned. - self.block_size = kv_cache_spec.block_size self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks self.has_prefill_checkpoint_blocks = ( diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 499b50a3a2..5791704b2b 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -158,6 +158,8 @@ class KVCacheSpec: # number of tokens in a block block_size: int + dcp_sharded: bool = field(default=False, kw_only=True) + """Whether DCP shards this cache's token positions across ranks.""" block_stride_alignment: int | None = field(default=None, kw_only=True) """Required byte alignment between physical blocks, including packed layers.""" @@ -481,6 +483,7 @@ class HiSparseResidentSpec(KVCacheSpec): @dataclass(frozen=True, kw_only=True) class AttentionSpec(KVCacheSpec): + dcp_sharded: bool = True num_kv_heads: int head_size: int dtype: torch.dtype @@ -536,7 +539,9 @@ class AttentionSpec(KVCacheSpec): def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int: parallel_config = vllm_config.parallel_config - kv_shard_count = parallel_config.decode_context_parallel_size + kv_shard_count = ( + parallel_config.decode_context_parallel_size if self.dcp_sharded else 1 + ) return cdiv(max_len, self.block_size * kv_shard_count) @@ -568,7 +573,7 @@ class FullAttentionSpec(AttentionSpec): def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: max_model_len = vllm_config.model_config.max_model_len dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size - if dcp_world_size > 1: + if self.dcp_sharded and dcp_world_size > 1: max_model_len = cdiv(max_model_len, dcp_world_size) return cdiv(max_model_len, self.block_size) * self.page_size_bytes @@ -612,6 +617,7 @@ class FullAttentionSpec(AttentionSpec): head_size_v=specs[0].head_size_v, dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -698,6 +704,7 @@ class MLAAttentionSpec(FullAttentionSpec): head_size=specs[0].head_size, dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -760,6 +767,7 @@ class RSWASpec(FullAttentionSpec): head_size_v=base.head_size_v, dtype=base.dtype, kv_quant_mode=base.kv_quant_mode, + dcp_sharded=base.dcp_sharded, page_size_padded=base.page_size_padded, num_head_slots=base.num_head_slots, state_content_bytes=base.state_content_bytes, @@ -850,9 +858,10 @@ class SlidingWindowSpec(AttentionSpec): return cdiv(num_tokens, self.block_size) + 1 def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: - assert vllm_config.parallel_config.decode_context_parallel_size == 1, ( - "DCP not support sliding window." - ) + assert ( + not self.dcp_sharded + or vllm_config.parallel_config.decode_context_parallel_size == 1 + ), "DCP not support sliding window." max_blocks = self.max_admission_blocks_per_request( max_in_flight_tokens=vllm_config.max_in_flight_tokens, max_model_len=vllm_config.model_config.max_model_len, @@ -950,6 +959,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): extra_retained_set = set(spec.extra_retained_tokens for spec in specs) bounded_replay_set = set(spec.bounded_replay for spec in specs) block_stride_alignment_set = {spec.block_stride_alignment for spec in specs} + assert len({spec.dcp_sharded for spec in specs}) == 1 assert ( len(cache_dtype_str_set) == 1 and len(tokens_per_state_set) == 1 @@ -969,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): num_kv_heads=specs[0].num_kv_heads, head_size=specs[0].head_size, dtype=specs[0].dtype, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -1185,6 +1196,7 @@ class SinkFullAttentionSpec(FullAttentionSpec): block_stride_alignment=specs[0].block_stride_alignment, dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -1217,6 +1229,11 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): kv_cache_specs: dict[str, KVCacheSpec] + def __post_init__(self): + super().__post_init__() + if self.kv_cache_specs: + object.__setattr__(self, "dcp_sharded", self.first_spec.dcp_sharded) + @property def prefix_cacheable(self) -> bool: return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values()) @@ -1270,6 +1287,8 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): if len(block_sizes) > 1: # Different block sizes, not uniform. return False + if len({spec.dcp_sharded for spec in kv_cache_specs.values()}) > 1: + return False first_spec = next(iter(kv_cache_specs.values())) return first_spec.is_uniform_with_collection(kv_cache_specs) diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 96bfb35a67..f70afaa343 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -4,7 +4,7 @@ import contextlib from collections.abc import Iterable -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field, fields, replace from enum import Enum, auto from typing import TYPE_CHECKING, Any @@ -27,13 +27,13 @@ from vllm.v1.core.kv_cache_coordinator import ( from vllm.v1.core.kv_cache_utils import ( BlockHashWithGroupId, ExternalBlockHash, - dcp_world_size_for_kv_cache_spec, generate_block_hash_extra_keys, get_block_hash, get_group_id, make_block_hash_with_group_id, maybe_convert_block_hash, resolve_block_hashes, + resolve_dcp_kv_block_size, ) from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import ( @@ -48,6 +48,11 @@ from vllm.v1.simple_kv_offload.metadata import ( SimpleCPUOffloadMetadata, SimpleCPUOffloadWorkerMetadata, ) +from vllm.v1.simple_kv_offload.metrics import ( + OUTCOME_TO_FIELD, + MetricName, + SimpleCPUOffloadStats, +) if TYPE_CHECKING: from vllm.v1.core.kv_cache_manager import KVCacheBlocks @@ -118,7 +123,8 @@ class BoundaryStoreStats: symptom is a lower cache hit rate with nothing in the logs, so each decline reason is counted and exposed through ``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by - ``reset()``; not otherwise cleared. + ``reset()``, which carries any not-yet-drained deltas; not otherwise + cleared. """ published: int = 0 @@ -143,6 +149,7 @@ class SimpleCPUOffloadScheduler: hash_block_size: int, lazy_offload: bool = False, disk_capacity_bytes: int = 0, + use_page_cache: bool = False, ): self.vllm_config = vllm_config self.kv_cache_config = kv_cache_config @@ -169,6 +176,13 @@ class SimpleCPUOffloadScheduler: self.cpu_kv_cache_config.prefix_cacheable_group_ids ) self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU + # Must track kv_event_medium. + self._info_labelvalues: tuple[str, ...] = ( + "disk" if disk_capacity_bytes > 0 else "cpu", + str(use_page_cache).lower(), + str(lazy_offload).lower(), + str(self.num_cpu_blocks), + ) # Find the full attention kv group for prefix cache matching. self.fa_gidx = -1 for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups): @@ -251,6 +265,9 @@ class SimpleCPUOffloadScheduler: self._load_event_counter: int = 0 self._store_event_counter: int = 0 self.boundary_store_stats = BoundaryStoreStats() + # Interval stats state drained by get_stats() + self._boundary_stats_snapshot = BoundaryStoreStats() + self._interval_load_blocks_completed = 0 # For TP/PP: track partial store completions across steps. # Events must be reported by all world_size workers before considered complete. @@ -302,11 +319,7 @@ class SimpleCPUOffloadScheduler: target = 0 for g in kv_cache_config.prefix_cacheable_groups: spec = g.kv_cache_spec - # Only full attention is sharded across DCP ranks; replicated specs - # (mamba, sliding window, chunked-local) keep their own block size. - block_size = spec.block_size * dcp_world_size_for_kv_cache_spec( - spec, cp_world_size - ) + block_size = resolve_dcp_kv_block_size(spec, cp_world_size) if isinstance(spec, MambaSpec): target += 2 elif isinstance(spec, SlidingWindowSpec): @@ -955,6 +968,42 @@ class SimpleCPUOffloadScheduler: def get_boundary_store_stats(self) -> BoundaryStoreStats: return replace(self.boundary_store_stats) + def get_stats(self) -> SimpleCPUOffloadStats: + """Drain per-step stats for the connector's stats hooks.""" + stats = SimpleCPUOffloadStats() + + current = self.boundary_store_stats + for outcome, field_name in OUTCOME_TO_FIELD.items(): + delta = getattr(current, field_name) - getattr( + self._boundary_stats_snapshot, field_name + ) + if delta > 0: + stats.increase_counter(MetricName.SAVE_OUTCOMES, delta, (outcome,)) + self._boundary_stats_snapshot = replace(current) + + if self._interval_load_blocks_completed: + stats.increase_counter( + MetricName.LOAD_BLOCKS, + self._interval_load_blocks_completed, + ) + self._interval_load_blocks_completed = 0 + + stats.set_gauge( + MetricName.USED_BLOCKS, + self.num_cpu_blocks - self.cpu_block_pool.get_num_free_blocks(), + ) + pending = sum( + len(t.cpu_block_ids) + for t in ( + *self._store_event_to_blocks.values(), + *self._pending_finished_stores, + *self._abandoned_store_event_to_blocks.values(), + ) + ) + stats.set_gauge(MetricName.PENDING_STORE_BLOCKS, pending) + stats.set_gauge(MetricName.INFO, 1, self._info_labelvalues) + return stats + def update_connector_output(self, connector_output: KVConnectorOutput) -> None: """Handle async transfer completions from worker. @@ -965,7 +1014,9 @@ class SimpleCPUOffloadScheduler: """ # --- Load completions --- for req_id in list(connector_output.finished_recving or []): - self._cleanup_load_request(req_id) + completed_blocks = self._cleanup_load_request(req_id) + if completed_blocks: + self._interval_load_blocks_completed += completed_blocks # --- Store completions --- meta = connector_output.kv_connector_worker_meta @@ -1270,18 +1321,22 @@ class SimpleCPUOffloadScheduler: if blocks_to_free: self.cpu_block_pool.free_blocks(blocks_to_free) - def _cleanup_load_request(self, req_id: str) -> None: + def _cleanup_load_request(self, req_id: str) -> int: """Release all load resources for a request. Shared between request_finished() and update_connector_output() paths. Removes the request from _reqs_to_load, cleans up event mappings, and frees CPU/GPU touch refs. + + Returns the number of blocks in the load if it had been issued to + the worker, else 0 (never-issued loads are not counted as completed + by the caller). """ state = self._reqs_to_load.pop(req_id, None) if state is None: state = self._abandoned_reqs_to_load.pop(req_id, None) if state is None: - return + return 0 # Remove from load event mapping (only this req, not whole event) if state.load_event is not None: reqs = self._load_event_to_reqs.get(state.load_event) @@ -1304,6 +1359,10 @@ class SimpleCPUOffloadScheduler: for bid in state.transfer_meta.gpu_block_ids ) + if state.load_event is None: + return 0 + return len(state.transfer_meta.gpu_block_ids) + def _cleanup_store_request(self, req_id: str) -> None: """Release store metadata for a request. @@ -1363,7 +1422,20 @@ class SimpleCPUOffloadScheduler: if event_idx in self._abandoned_store_event_to_blocks } self._cursor = None - self.boundary_store_stats = BoundaryStoreStats() + # Seed the fresh counters with outcomes since the last drain so a + # mid-interval reset() does not drop them from the next report. + undrained = BoundaryStoreStats( + **{ + f.name: max( + 0, + getattr(self.boundary_store_stats, f.name) + - getattr(self._boundary_stats_snapshot, f.name), + ) + for f in fields(BoundaryStoreStats) + } + ) + self.boundary_store_stats = undrained + self._boundary_stats_snapshot = BoundaryStoreStats() # NOTE: _load_event_counter / _store_event_counter are not # reset as they are monotonic and must stay ahead of the workers # high-water marks to avoid event index collisions diff --git a/vllm/v1/simple_kv_offload/metrics.py b/vllm/v1/simple_kv_offload/metrics.py new file mode 100644 index 0000000000..9017d40cfd --- /dev/null +++ b/vllm/v1/simple_kv_offload/metrics.py @@ -0,0 +1,284 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Prometheus metrics for SimpleCPUOffloadConnector. + +The stats payload uses the same self-describing wire format as the +offloading connector, so it survives IPC serialization without the +metric classes on the receiving side. +""" + +from dataclasses import dataclass +from typing import Any + +from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) + + +class MetricName: + """Metric names for SimpleCPUOffloadConnector.""" + + SAVE_OUTCOMES = "vllm:simple_kv_offload_save_outcomes_total" + LOAD_BLOCKS = "vllm:simple_kv_offload_load_blocks_total" + USED_BLOCKS = "vllm:simple_kv_offload_used_blocks" + PENDING_STORE_BLOCKS = "vllm:simple_kv_offload_pending_store_blocks" + INFO = "vllm:simple_kv_offload_info" + + +# outcome label value -> BoundaryStoreStats field name. +# ``published`` is the sum of all outcomes and is not labeled. +OUTCOME_TO_FIELD = { + "stored": "stored", + "dropped_cpu_full": "dropped_cpu_full", + "dropped_request_gone": "dropped_request_gone", + "dropped_null_block": "dropped_null_block", + "dropped_not_hashed": "dropped_not_hashed", + "skipped_already_cached": "skipped_already_cached", + "skipped_in_flight": "skipped_in_flight", +} + +INFO_LABELS = ["backend", "page_cache", "lazy_offload", "capacity_blocks"] + + +class _MetricType: + """Type tags embedded in the serialized stats payload.""" + + COUNTER = "counter" + GAUGE = "gauge" + + +class _StatsKey: + """Top-level keys in the serialized stats dict.""" + + # Maps metric name -> _MetricType value + TYPES = "types" + # Maps metric name -> {label values tuple -> observed value} + DATA = "data" + + +@dataclass +class SimpleCPUOffloadStats(KVConnectorStats): + """Stats payload for SimpleCPUOffloadConnector. + + The ``data`` dict is structured as:: + + { + _StatsKey.TYPES: {name: _MetricType.*, ...}, + _StatsKey.DATA: {name: {labelvalues: value, ...}, ...}, + } + + Counter values are summed when aggregating, gauge values keep the + latest snapshot, and unlabeled metrics use ``()`` as their + labelvalues tuple. + """ + + def __post_init__(self): + if _StatsKey.DATA not in self.data: + self.reset() + + def reset(self): + self.data: dict[str, Any] = { + _StatsKey.TYPES: {}, + _StatsKey.DATA: {}, + } + + @property + def _types(self) -> dict[str, str]: + return self.data[_StatsKey.TYPES] + + @property + def _values(self) -> dict[str, Any]: + return self.data[_StatsKey.DATA] + + def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats": + if other.is_empty(): + return self + assert isinstance(other, SimpleCPUOffloadStats) + for key, other_label_values in other._values.items(): + type_str = other._types.get(key) + if type_str is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + self._types.setdefault(key, type_str) + current_label_values = self._values.setdefault(key, {}) + for labelvalues, value in other_label_values.items(): + if type_str == _MetricType.COUNTER: + assert isinstance(value, int | float) + current_label_values[labelvalues] = ( + current_label_values.get(labelvalues, 0) + value + ) + elif type_str == _MetricType.GAUGE: + assert isinstance(value, int | float) + current_label_values[labelvalues] = value + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) + return self + + def reduce(self) -> dict[str, int | float]: + """Reduce the interval observations to a flat summary for logging.""" + return_dict: dict[str, int | float] = {} + for key, label_value_map in self._values.items(): + if key == MetricName.INFO: + # Constant config fingerprint; exclude from the log summary. + continue + type_str = self._types.get(key) + if type_str is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + for labelvalues, value in label_value_map.items(): + key_with_labels = f"{key}:{labelvalues}" if labelvalues else key + if type_str in (_MetricType.COUNTER, _MetricType.GAUGE): + assert isinstance(value, int | float) + return_dict[key_with_labels] = value + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) + return return_dict + + def is_empty(self) -> bool: + return not self.data.get(_StatsKey.DATA) + + def increase_counter( + self, + counter_name: str, + counter_increase_value: int | float = 1, + labelvalues: tuple[str, ...] = (), + ) -> None: + """Increase a counter on the stats payload.""" + self._types.setdefault(counter_name, _MetricType.COUNTER) + counter_values = self._values.setdefault(counter_name, {}) + counter_values[labelvalues] = ( + counter_values.get(labelvalues, 0) + counter_increase_value + ) + + def set_gauge( + self, + gauge_name: str, + gauge_value: int | float, + labelvalues: tuple[str, ...] = (), + ) -> None: + """Set a gauge snapshot on the stats payload.""" + self._types.setdefault(gauge_name, _MetricType.GAUGE) + gauge_values = self._values.setdefault(gauge_name, {}) + gauge_values[labelvalues] = gauge_value + + +class SimpleCPUOffloadPromMetrics(KVConnectorPromMetrics): + """Prometheus registration and recording for SimpleCPUOffloadConnector.""" + + def __init__( + self, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ): + super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) + # (engine_idx, metric_name, labelvalues) -> bound child metric + self._metrics: dict[tuple[int, str, tuple[str, ...]], PromMetricT] = {} + + self._counter_save_outcomes = self._counter_cls( + name="vllm:simple_kv_offload_save_outcomes_total", + documentation=( + "Store admission decisions in SimpleCPUOffloadConnector, by " + "outcome. Outcomes classify eager-mode boundary hand-off " + "stores; lazy-mode stores are not classified." + ), + labelnames=self._labelnames + ["outcome"], + ) + self._counter_load_blocks = self._counter_cls( + name="vllm:simple_kv_offload_load_blocks_total", + documentation=( + "KV blocks restored to GPU from the offload pool, counted " + "when a load finishes. This is prefill work avoided " + "through offload cache hits." + ), + labelnames=self._labelnames, + ) + self._gauge_used_blocks = self._gauge_cls( + name="vllm:simple_kv_offload_used_blocks", + documentation=( + "Offload-pool blocks currently pinned by in-flight " + "transfers or cache hits (capacity minus free). Evictable " + "cached blocks are not counted; capacity_blocks is on " + "vllm:simple_kv_offload_info." + ), + labelnames=self._labelnames, + multiprocess_mode="mostrecent", + ) + self._gauge_pending_store_blocks = self._gauge_cls( + name="vllm:simple_kv_offload_pending_store_blocks", + documentation=( + "Offload blocks in pending stores: queued for dispatch, " + "issued to workers, or awaiting release after a cache " + "reset. A persistently growing value indicates a stuck " + "transfer." + ), + labelnames=self._labelnames, + multiprocess_mode="mostrecent", + ) + self._gauge_info = self._gauge_cls( + name="vllm:simple_kv_offload_info", + documentation=( + "SimpleCPUOffloadConnector deployment facts. Value is " + "always 1. backend is cpu or disk; page_cache and " + "lazy_offload are true/false; capacity_blocks is the " + "offload pool size." + ), + labelnames=self._labelnames + INFO_LABELS, + multiprocess_mode="mostrecent", + ) + + self._defs: dict[str, PromMetricT] = { + MetricName.SAVE_OUTCOMES: self._counter_save_outcomes, + MetricName.LOAD_BLOCKS: self._counter_load_blocks, + MetricName.USED_BLOCKS: self._gauge_used_blocks, + MetricName.PENDING_STORE_BLOCKS: self._gauge_pending_store_blocks, + MetricName.INFO: self._gauge_info, + } + self._num_label_values: dict[str, int] = { + MetricName.SAVE_OUTCOMES: 1, + MetricName.LOAD_BLOCKS: 0, + MetricName.USED_BLOCKS: 0, + MetricName.PENDING_STORE_BLOCKS: 0, + MetricName.INFO: len(INFO_LABELS), + } + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + """Observe transfer statistics.""" + metric_types = transfer_stats_data.get(_StatsKey.TYPES, {}) + metric_data = transfer_stats_data.get(_StatsKey.DATA, {}) + for key, label_value_map in metric_data.items(): + prom_metric = self._defs.get(key) + if prom_metric is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + type_str = metric_types.get(key) + for labelvalues, value in label_value_map.items(): + if len(labelvalues) != self._num_label_values[key]: + raise AssertionError( + f"Metric {key} expects " + f"{self._num_label_values[key]} labels, " + f"got {len(labelvalues)}" + ) + child = self._metrics.get((engine_idx, key, labelvalues)) + if child is None: + engine_labelvalues = self.per_engine_labelvalues[engine_idx] + child = prom_metric.labels( + *(engine_labelvalues + list(labelvalues)) + ) + self._metrics[(engine_idx, key, labelvalues)] = child + if type_str == _MetricType.COUNTER: + assert isinstance(value, int | float) + child.inc(value) + elif type_str == _MetricType.GAUGE: + assert isinstance(value, int | float) + child.set(value) + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) diff --git a/vllm/v1/spec_decode/draft_model.py b/vllm/v1/spec_decode/draft_model.py index 08542f03a1..17be2732e6 100644 --- a/vllm/v1/spec_decode/draft_model.py +++ b/vllm/v1/spec_decode/draft_model.py @@ -81,13 +81,20 @@ class DraftModelProposer(SpecDecodeBaseProposer): def _create_draft_vllm_config(self) -> VllmConfig: base = super()._create_draft_vllm_config() spec = self.speculative_config + target_parallel = base.parallel_config return replace( base, quant_config=None, parallel_config=replace( spec.draft_parallel_config, - rank=self.vllm_config.parallel_config.rank, + rank=target_parallel.rank, + decode_context_parallel_size=( + target_parallel.decode_context_parallel_size + ), + cp_kv_cache_interleave_size=target_parallel.cp_kv_cache_interleave_size, + dcp_comm_backend=target_parallel.dcp_comm_backend, + dcp_q_replicate=target_parallel.dcp_q_replicate, ), model_config=spec.draft_model_config, ) diff --git a/vllm/v1/worker/cp_utils.py b/vllm/v1/worker/cp_utils.py index 79651759aa..bcf56dc0bf 100644 --- a/vllm/v1/worker/cp_utils.py +++ b/vllm/v1/worker/cp_utils.py @@ -40,6 +40,8 @@ def check_attention_cp_compatibility( layer_impl = getattr(layer, "impl", None) if layer_impl is None: continue + if not check_pcp and layer_impl.dcp_world_size == 1: + continue if vllm_config.speculative_config is not None and interleave_size > 1: assert layer_impl.supports_mtp_with_cp_non_trivial_interleave_size, ( "MTP with cp_kv_cache_interleave_size > 1 is not " diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 096d9b99aa..8cf3ba98dd 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -27,6 +27,7 @@ class BlockTables: cp_rank: int = 0, cp_interleave: int = 1, slot_mapping_enabled: list[bool] | None = None, + dcp_sharded: list[bool] | None = None, ): self.block_sizes = block_sizes self.kernel_block_sizes = kernel_block_sizes @@ -44,6 +45,10 @@ class BlockTables: slot_mapping_enabled = [True] * self.num_kv_cache_groups assert len(slot_mapping_enabled) == self.num_kv_cache_groups self._slot_mapping_enabled = slot_mapping_enabled + if dcp_sharded is None: + dcp_sharded = [True] * self.num_kv_cache_groups + assert len(dcp_sharded) == self.num_kv_cache_groups + self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device) self.blocks_per_kv_block = [ bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes) @@ -210,6 +215,7 @@ class BlockTables: self.block_sizes_tensor, self.kernel_block_sizes_tensor, self.slot_mapping_enabled, + self.dcp_sharded, slot_mappings, slot_mappings.stride(0), self.cp_rank, @@ -283,6 +289,7 @@ def _compute_slot_mappings_kernel( block_sizes, # [num_kv_cache_groups] kernel_block_sizes, # [num_kv_cache_groups] slot_mapping_enabled, # [num_kv_cache_groups] + dcp_sharded, # [num_kv_cache_groups] slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens] slot_mappings_stride, cp_rank, @@ -312,6 +319,8 @@ def _compute_slot_mappings_kernel( kv_block_size = tl.load(block_sizes + group_id) kernel_block_size = tl.load(kernel_block_sizes + group_id) mapping_enabled = tl.load(slot_mapping_enabled + group_id) + if CP_SIZE != 1: + sharded = tl.load(dcp_sharded + group_id) req_state_idx = tl.load(idx_mapping + batch_idx) # idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns @@ -337,6 +346,8 @@ def _compute_slot_mappings_kernel( remainder = virtual_block_offsets % CP_INTERLEAVE local_offsets = rounds * CP_INTERLEAVE + remainder local_positions = virtual_block_indices * kv_block_size + local_offsets + local_positions = tl.where(sharded, local_positions, positions) + is_local = ~sharded | is_local block_indices = tl.where( mapping_enabled, local_positions // kernel_block_size, 0 diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 61ceaeef2e..9f25425ab6 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -21,6 +21,7 @@ import gc import time from contextlib import AbstractContextManager from copy import deepcopy +from dataclasses import replace from typing import Any, NamedTuple import numpy as np @@ -561,7 +562,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): return encoder_runner.get_encoder_timing_stats() def get_kv_cache_spec(self): - return get_kv_cache_spec(self.vllm_config) + kv_cache_spec = get_kv_cache_spec(self.vllm_config) + # A draft running at DCP=1 under a DCP target keeps a replicated cache. + if ( + isinstance(self.speculator, DraftModelSpeculator) + and self.speculator.dcp_size == 1 + and self.dcp_size > 1 + ): + for name in self.speculator.draft_attn_layer_names & kv_cache_spec.keys(): + kv_cache_spec[name] = replace(kv_cache_spec[name], dcp_sharded=False) + return kv_cache_spec def initialize_kv_cache( self, @@ -587,6 +597,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): block_sizes = [] max_num_blocks_per_group = [] slot_mapping_enabled = [] + dcp_sharded = [] for kv_cache_group in kv_cache_config.kv_cache_groups: spec = kv_cache_group.kv_cache_spec block_sizes.append(spec.block_size) @@ -594,6 +605,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec ) slot_mapping_enabled.append(layer_spec.uses_slot_mapping) + dcp_sharded.append(spec.dcp_sharded) # Let each cache type account for CP. Attention KV is DCP-sharded, # while Mamba/GDN recurrent state is replicated across DCP ranks. max_num_blocks = spec.max_num_blocks_per_req( @@ -657,6 +669,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): device=self.device, kernel_block_sizes=self.kernel_block_sizes, slot_mapping_enabled=slot_mapping_enabled, + dcp_sharded=dcp_sharded, cp_size=self.dcp_size, cp_rank=self.dcp_rank, cp_interleave=self.cp_interleave, diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index aff8f04cf3..de49fcffb9 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -145,7 +145,10 @@ class PCPManager: if speculative_config is not None: if speculative_config.use_dspark(): dcp_size = parallel_config.decode_context_parallel_size - if dcp_size not in (1, pcp_size): + if ( + dcp_size not in (1, pcp_size) + and speculative_config.draft_model_config.use_mla + ): raise NotImplementedError( "MRV2 PCP DSpark requires DCP=1 or DCP=PCP; got " f"DCP={dcp_size}, PCP={pcp_size}." diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index cdcdd5fcba..57110e0b05 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -37,12 +37,18 @@ class DFlashSpeculator(DraftModelSpeculator): def __init__(self, vllm_config: VllmConfig, device: torch.device): parallel_config = vllm_config.parallel_config - if parallel_config.prefill_context_parallel_size > 1: - vllm_config = copy.copy(vllm_config) - vllm_config.parallel_config = replace( - parallel_config, - prefill_context_parallel_size=1, - ) + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + vllm_config = copy.copy(vllm_config) + vllm_config.parallel_config = replace( + parallel_config, + prefill_context_parallel_size=1, + decode_context_parallel_size=( + parallel_config.decode_context_parallel_size + if speculative_config.draft_model_config.use_mla + else 1 + ), + ) super().__init__(vllm_config, device) self.hidden_states = torch.zeros( @@ -427,7 +433,7 @@ class DFlashSpeculator(DraftModelSpeculator): self.block_tables.input_block_tables[gid], self.block_tables.kernel_block_sizes[gid], self.block_tables.cp_rank, - self.block_tables.cp_size, + self.dcp_size, self.block_tables.cp_interleave, self.parallel_drafting_token_id, self.num_query_per_req, diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index d1d6d888aa..6f88febb66 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -138,6 +138,7 @@ class DraftModelSpeculator(BaseSpeculator): # DP configuration self.dp_size = vllm_config.parallel_config.data_parallel_size self.dp_rank = vllm_config.parallel_config.data_parallel_rank + self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size self.eplb_state: EplbState | None = None @@ -332,14 +333,14 @@ class DraftModelSpeculator(BaseSpeculator): out=draft_seq_lens_cpu_upper_bound[:num_reqs], ) draft_seq_lens_cpu_upper_bound[:num_reqs].clamp_(max=self.max_model_len) - if dcp_local_seq_lens is None and self.block_tables.cp_size > 1: + if dcp_local_seq_lens is None and self.dcp_size > 1: # Draft steps advance and rewind their own global sequence lengths, # so the target model's DCP-local lengths may already be stale. dcp_local_seq_lens = prepare_dcp_local_seq_lens( self.input_buffers.dcp_local_seq_lens, self.input_buffers.seq_lens, num_reqs, - self.block_tables.cp_size, + self.dcp_size, self.block_tables.cp_rank, self.block_tables.cp_interleave, ) diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index 2a1a3534c9..4707e4f2ca 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -817,6 +817,7 @@ class InputBatch: self.allowed_token_ids_mask_cpu_tensor[empty_index] = ( self.allowed_token_ids_mask_cpu_tensor[last_req_index] ) + self.allowed_token_ids_mask_cpu_tensor[last_req_index].fill_(False) bad_words_token_ids = self.bad_words_token_ids.pop(last_req_index, None) if bad_words_token_ids is not None: diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index def92f601d..778d71bdbc 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -801,11 +801,20 @@ class Worker(WorkerBase): # related to kv cache connector (e.g. kv cache sharing layers). ensure_kv_transfer_initialized(self.vllm_config, kv_cache_config) + # If the connector provides a custom memory pool (e.g. Mooncake + # NVLink/BAREX), use it for KV cache allocation; otherwise fall + # back to the standard CuMem pool. + mem_pool_context = ( + get_kv_transfer_group().get_mem_pool_context() + if has_kv_transfer_group() + else None + ) + if mem_pool_context is None: + mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache") + self.model_runner.initialize_kv_cache( kv_cache_config, - kv_cache_allocation_context=self._maybe_get_memory_pool_context( - tag="kv_cache" - ), + kv_cache_allocation_context=mem_pool_context, ) # Build KV-zero metadata outside the CuMem pool so the bookkeeping