Merge branch 'main' into gdn-recoverssm

This commit is contained in:
Jürgen Schmied
2026-09-27 08:01:32 +02:00
committed by GitHub
66 changed files with 2899 additions and 367 deletions
+1 -1
View File
@@ -13,7 +13,7 @@ else()
FetchContent_Declare(
flashkda
GIT_REPOSITORY https://github.com/vllm-project/FlashKDA.git
GIT_TAG b59532f1f464fbd536272780e30df5bf6a2ccc02
GIT_TAG 17a037d98da546deb4591e967cf961a43c034d8b
GIT_PROGRESS TRUE
GIT_SUBMODULES cutlass
)
+1 -1
View File
@@ -144,7 +144,7 @@ When batch invariance is enabled, vLLM:
1. Uses deterministic kernel implementations for attention and other operations
2. Ensures consistent numerical behavior across different batch sizes
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode)
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant)
4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out
(not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes).
@@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [
"path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py",
"key": "metrics-nixl",
},
{
"path": "vllm/v1/simple_kv_offload/metrics.py",
"key": "metrics-simple-kv-offload",
},
{"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"},
]
+24
View File
@@ -45,6 +45,30 @@ The following metrics are exposed:
--8<-- "gen:metrics-nixl"
## Simple CPU Offload Connector Metrics
These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector
is configured (e.g. `--kv-transfer-config='{"kv_connector":
"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config":
{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are
updated once per engine step.
Caveats to keep in mind when interpreting them:
- A "completed" store means the write syscalls returned; it is **not**
fsync-durable, and the disk backend's file is process-lifetime scratch,
unlinked at startup and shutdown.
- `save_outcomes_total` classifies eager-mode boundary hand-off stores;
lazy-mode stores are not classified.
- `used_blocks` counts blocks pinned by in-flight transfers or cache hits;
warm cached blocks that are evictable are not counted. Use the
`capacity_blocks` label of `simple_kv_offload_info` as the denominator.
- Counters and gauges are quantized to engine steps. They are reported by the
scheduler process and reflect engine-wide logical block counts, not values
pooled from individual tensor-parallel workers.
--8<-- "gen:metrics-simple-kv-offload"
## Model Flops Utilization (MFU) Performance Metrics
These metrics are available via `--enable-mfu-metrics`:
+44 -52
View File
@@ -566,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected
def test_aiter_moe_situv2_syncs_aiter_a4w4_env(
monkeypatch: pytest.MonkeyPatch,
):
"""SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
mp.delenv("AITER_SITUV2_A8W4", raising=False)
mp.delenv("AITER_SITUV2_A4W4", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
assert "AITER_SITUV2_A8W4" not in os.environ
def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4(
monkeypatch: pytest.MonkeyPatch,
):
"""Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
mp.delenv("AITER_SITUV2_A8W4", raising=False)
mp.delenv("AITER_SITUV2_A4W4", raising=False)
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
assert "AITER_SITUV2_A8W4" not in os.environ
def test_aiter_moe_situv2_clears_aiter_env_when_disabled(
@pytest.mark.parametrize(
"value,expected_act,expected_env",
[
# Unset / auto / legacy "1" all mean the a4w4 default.
(None, "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("0", "a16w4", {}),
("a16w4", "a16w4", {}),
("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}),
],
)
def test_aiter_moe_situv2_activation_syncs_aiter_env(
value: str | None,
expected_act: str,
expected_env: dict[str, str],
monkeypatch: pytest.MonkeyPatch,
):
"""Each SiTUv2 activation choice sets exactly its AITER dispatch env."""
import os
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
# Stale AITER overrides must be replaced by the vLLM selection.
mp.setenv("AITER_SITUV2_A8W4", "1")
mp.setenv("AITER_SITUV2_A4W4", "1")
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0")
if value is None:
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
else:
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value)
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act
assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is (
expected_act == "a8w4"
)
assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4")
for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"):
assert os.environ.get(name) == expected_env.get(name)
assert "AITER_SITUV2_A4W4" not in os.environ
def test_aiter_moe_situv2_rejects_unknown_activation(
monkeypatch: pytest.MonkeyPatch,
):
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8")
_reload_envs()
import vllm.envs as envs
with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"):
_ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
@pytest.mark.parametrize("moe_padding", [True, False])
@@ -21,7 +21,12 @@ Usage:
import pytest
import torch
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
deepgemm_post_process_fp8_weight_block,
)
from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import fp8_einsum
# -- Default dimensions matching DeepSeek V3/V4 --------------------------
HEAD_DIM = 512
@@ -33,6 +38,49 @@ FP8_DTYPE = torch.float8_e4m3fn
EPS = 1e-10
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103"
)
@pytest.mark.parametrize("groups", [2, 4])
@pytest.mark.parametrize("tokens", [1, 6, 18, 32])
def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens):
"""Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize."""
from flashinfer import mxfp8_quantize
from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL
torch.manual_seed(0)
n = groups * 1024
x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128]
positions = torch.randint(0, 128, (tokens,), device="cuda")
rope = make_cos_sin_cache(128, device="cuda")
wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE)
scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float())
wq, ws = deepgemm_post_process_fp8_weight_block(
wq, scale, (1, 32), False, True, groups
)
q, sf = _FUSED_WO_A_KERNEL(
x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws
)
a, sa = fused_inv_rope_fp8_quant(
x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True
)
y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16)
fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32))
ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl")
def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks]
return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1)
actual, expected = (
d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0)
for d, s in ((q, sf), (ref, ref_sf))
)
assert (actual - expected).norm() / expected.norm() < 3e-3
assert not rows(sf)[tokens:].any(), "padded scale rows must be zero"
# =========================================================================
# Helpers
# =========================================================================
@@ -942,8 +942,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
impl.o_sf_scale = None
impl.window_left = -1
impl.sinks = None
impl.cache_config = unittest.mock.Mock()
impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC
layer = unittest.mock.Mock(
_q_scale=torch.tensor(1.0),
@@ -959,6 +957,7 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
q_len_per_req=1,
)
attn_metadata = flashinfer_backend.FlashInferMetadata(
kv_cache_layout=KVCacheLayout.LBHNC,
num_actual_tokens=2,
slot_mapping=torch.empty(0, dtype=torch.int64),
q_data_type_prefill=torch.bfloat16,
@@ -101,6 +101,7 @@ def test_flash_attention_geometry_comes_from_the_group():
num_kv_heads=2,
head_size=64,
dtype=torch.bfloat16,
dcp_sharded=True,
)
with (
+143 -12
View File
@@ -1599,19 +1599,142 @@ def test_project_kv_cache_groups_to_worker():
assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"}
def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only():
dcp = 8
full = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
@pytest.mark.parametrize("sliding_window", [None, 256])
@pytest.mark.parametrize("disable_hybrid", [False, True])
@pytest.mark.parametrize("pcp_size", [1, 4])
def test_dcp_target_allocates_replicated_draft_independently(
monkeypatch, sliding_window, disable_hybrid, pcp_size
):
"""A draft must retain all positions even when the target shards them."""
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.parallel_config.decode_context_parallel_size = 4
config.parallel_config.prefill_context_parallel_size = pcp_size
config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hybrid
config.cache_config.block_size = 16
config.cache_config.kv_cache_layout = None
draft_args = dict(
block_size=16,
num_kv_heads=8,
head_size=64,
dtype=torch.bfloat16,
dcp_sharded=False,
)
mla = new_mla_spec()
mamba = new_mamba_spec()
uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla})
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1
draft = (
FullAttentionSpec(**draft_args)
if sliding_window is None
else SlidingWindowSpec(**draft_args, sliding_window=sliding_window)
)
specs = {"target": new_mla_spec(), "draft": draft}
layout = resolve_kv_cache_layout(config, [["LBHNC", "BLHNC"]], specs.values())
assert layout == KVCacheLayout.BLHNC
groups = get_kv_cache_groups(config, specs)
assert [group.layer_names for group in groups] == [["target"], ["draft"]]
widths = [g.kv_cache_spec.max_num_blocks_per_req(config, 1024) for g in groups]
assert widths == [16, 64]
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
config, groups, available_memory=16 * 1024 * 1024
)
scheduler_config = generate_scheduler_kv_cache_config([cache_config])
assert get_max_concurrency_for_kv_cache_config(config, cache_config) > 0
assert get_max_concurrency_for_kv_cache_config(config, cache_config) == (
get_max_concurrency_for_kv_cache_config(config, scheduler_config)
)
manager = KVCacheManager(
scheduler_config,
max_model_len=1024,
hash_block_size=16,
scheduler_block_size=64,
dcp_world_size=4,
pcp_world_size=pcp_size,
enable_caching=True,
)
assert manager.coordinator.group_block_sizes == (64, 16)
request = make_request("replicated-draft", [1] * 65, block_size=16, hash_fn=sha256)
blocks = manager.allocate_slots(request, 65)
assert blocks is not None
assert [len(group) for group in blocks.blocks] == [2, 5]
manager.cache_blocks(request, 64)
cached_request = make_request(
"cached-draft", [1] * 65, block_size=16, hash_fn=sha256
)
cached_blocks, num_cached, _ = manager.get_computed_blocks(cached_request)
assert num_cached == 64
assert [len(group) for group in cached_blocks.blocks] == [1, 4]
@pytest.mark.parametrize("use_mla", [False, True])
def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.parallel_config.decode_context_parallel_size = 4
draft = replace(
new_mla_spec() if use_mla else new_kv_cache_spec(),
block_size=16,
dcp_sharded=False,
)
merged = type(draft).merge([draft, draft])
wrapped = UniformTypeKVCacheSpecs.from_specs({"draft": merged})
assert wrapped is not None and not wrapped.dcp_sharded
assert merged.max_num_blocks_per_req(config, 1024) == 64
assert merged.max_memory_usage_bytes(config) == 64 * draft.page_size_bytes
assert kv_cache_utils.resolve_dcp_kv_block_size(merged, 4) == 16
with pytest.raises(AssertionError):
type(draft).merge([draft, replace(draft, dcp_sharded=True)])
@pytest.mark.parametrize("with_draft", [False, True])
@pytest.mark.parametrize("indexer_alignment", [None, 512])
def test_sparse_mla_preserves_physical_row_addressing(with_draft, indexer_alignment):
from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
from vllm.v1.worker.utils import allocate_kv_cache
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.cache_config.kv_cache_layout = "BLHNC"
config.parallel_config.decode_context_parallel_size = 4
common = dict(block_size=64, num_kv_heads=1, dtype=torch.uint8)
specs = {
"target": MLAAttentionSpec(
**common,
head_size=576,
state_content_bytes=656,
block_stride_alignment=656,
cache_dtype_str="fp8_ds_mla",
is_index_group_leader=True,
),
"indexer": MLAAttentionSpec(
**common,
head_size=132,
cache_role=SparseCacheRole.INDEXER,
block_stride_alignment=indexer_alignment,
),
"draft": SlidingWindowSpec(
block_size=64,
num_kv_heads=64,
head_size=64,
dtype=torch.bfloat16,
sliding_window=2048,
dcp_sharded=False,
),
}
if not with_draft:
del specs["draft"]
groups = get_kv_cache_groups(config, specs)
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
config, groups, available_memory=8 * 1024 * 1024
)
caches = allocate_kv_cache(
cache_config, torch.device("cpu"), KVCacheLayout.BLHNC, [64] * len(groups)
)
cache = caches["target"].squeeze(1)
if indexer_alignment is not None:
assert caches["indexer"].stride(0) % indexer_alignment == 0
rows, stride_rows = flat_kv_row_view(cache, 64)
cache[1, 0].fill_(7)
torch.testing.assert_close(rows[stride_rows], cache[1, 0])
assert torch.all(rows[stride_rows] == 7)
@pytest.mark.parametrize(
@@ -3815,6 +3938,8 @@ def test_unify_hybrid_kv_cache_specs():
"layer_1": before_spec_1,
"layer_2": before_spec_2,
}
kv_cache_spec["draft_layer_1"] = replace(before_spec_1, dcp_sharded=False)
kv_cache_spec["draft_layer_2"] = replace(before_spec_2, dcp_sharded=False)
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
expected_spec_1 = new_kv_cache_spec(block_size=64)
expected_spec_2 = new_kv_cache_spec(
@@ -3823,6 +3948,7 @@ def test_unify_hybrid_kv_cache_specs():
assert kv_cache_spec["layer_1"] == expected_spec_1
assert kv_cache_spec["layer_2"] == expected_spec_2
assert kv_cache_spec["layer_2"].page_size_bytes == 64 * 1024
assert kv_cache_spec["draft_layer_2"] == replace(expected_spec_2, dcp_sharded=False)
# 2. has_full_attention and has_chunked_local_attention
before_spec_1 = new_kv_cache_spec()
@@ -3874,6 +4000,11 @@ def test_unify_hybrid_kv_cache_specs():
with pytest.raises(ValueError):
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
# Replicated Mamba state still requires the hybrid cache manager.
kv_cache_spec = {"attention": new_kv_cache_spec(), "mamba": new_mamba_spec()}
with pytest.raises(ValueError):
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
def test_unify_kv_cache_spec_page_size_mamba():
"""Regression test for https://github.com/vllm-project/vllm/issues/43626.
+12 -7
View File
@@ -1336,12 +1336,12 @@ def make_kv_cache_config_three_types(
)
def test_prefix_cache_hit_uses_per_group_dcp_geometry():
@pytest.mark.parametrize("draft_sharded", [True, False])
def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
"""Prefix lookup must use each group's DCP size, not the process-wide one.
Target and draft MLA are both sharded (DCP=8); Mamba stays replicated
(DCP=1). Hits then align to the sharded full-attention block, not the
unsharded page size.
Target MLA is sharded; draft MLA can opt out and Mamba stays replicated.
Hits must align to the target block in either case.
"""
block_size = 16
dcp = 8
@@ -1362,6 +1362,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
KVCacheGroupSpec(
["draft_mla"],
MLAAttentionSpec(
dcp_sharded=draft_sharded,
block_size=block_size,
num_kv_heads=1,
head_size=1,
@@ -1388,10 +1389,10 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
)
target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers
assert target_mgr.dcp_world_size == dcp
assert draft_mgr.dcp_world_size == dcp
assert draft_mgr.dcp_world_size == (dcp if draft_sharded else 1)
assert mamba_mgr.dcp_world_size == 1
assert target_mgr.block_size == sharded_block
assert draft_mgr.block_size == sharded_block
assert draft_mgr.block_size == (sharded_block if draft_sharded else block_size)
assert mamba_mgr.block_size == block_size
hash_fn = sha256
@@ -1407,7 +1408,11 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 2 * sharded_block
assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16]
assert [len(group) for group in computed_blocks.blocks] == [
2,
2 if draft_sharded else 16,
16,
]
manager.free(req0)
manager.free(req1)
@@ -11,6 +11,7 @@ from vllm.config import CompilationConfig, VllmConfig, replace
from vllm.config.kernel import MoEBackend
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from ..utils import (
Messages,
@@ -196,6 +197,38 @@ def test_draft_model_engine_args_tensor_parallelism():
assert draft_config.quant_config is None
@pytest.mark.cpu_test
@pytest.mark.parametrize("dcp_size", [1, 2])
def test_draft_model_inherits_target_dcp_config(monkeypatch, dcp_size):
monkeypatch.setattr(current_platform, "device_count", lambda: 16)
target_config = EngineArgs(
model="Qwen/Qwen3-1.7B",
tensor_parallel_size=16,
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=16,
dcp_comm_backend="ag_rs",
distributed_executor_backend="mp",
speculative_config={
"model": "Qwen/Qwen3-0.6B",
"method": "draft_model",
"num_speculative_tokens": 3,
},
).create_engine_config()
proposer = object.__new__(DraftModelProposer)
proposer.vllm_config = target_config
proposer.speculative_config = target_config.speculative_config
draft_config = proposer._create_draft_vllm_config()
draft_parallel = draft_config.parallel_config
assert draft_parallel.decode_context_parallel_size == dcp_size
assert draft_parallel.cp_kv_cache_interleave_size == 16
assert draft_parallel.dcp_comm_backend == "ag_rs"
assert (
draft_parallel.dcp_q_replicate == target_config.parallel_config.dcp_q_replicate
)
assert draft_parallel is not target_config.parallel_config
assert target_config.parallel_config.decode_context_parallel_size == dcp_size
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
so we can test it without instantiating a full proposer."""
@@ -500,7 +500,7 @@ def test_decode_bench_connector_skips_null_padding():
def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
"""DCP scales full-attention blocks but not sliding-window blocks."""
"""DCP scales target blocks but not the replicated draft's window."""
block_size = 16
dcp_world_size = 2
vllm_config = create_vllm_config(
@@ -515,6 +515,7 @@ def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
dtype=torch.float32,
)
sliding_window = SlidingWindowSpec(
dcp_sharded=False,
block_size=block_size,
num_kv_heads=1,
head_size=1,
@@ -290,6 +290,8 @@ class _FakeKVTransferConfig:
class _FakeModelConfig:
model = "test-model"
use_mla = False
enable_sleep_mode = False
enable_cumem_allocator = False
def get_num_layers(self, parallel_config) -> int:
return 1
@@ -432,6 +434,64 @@ def _patch_worker_runtime(
monkeypatch.setattr(worker, "LookupKeyServer", MagicMock())
def test_create_mem_pool_returns_none_when_unconfigured():
assert worker.MooncakeStoreWorker._create_mem_pool({}, _FakeModelConfig()) is None
def test_create_mem_pool_rejects_unknown_pool():
with pytest.raises(
ValueError,
match="Unsupported custom_mem_pool='UNKNOWN'",
):
worker.MooncakeStoreWorker._create_mem_pool(
{"custom_mem_pool": "unknown"}, _FakeModelConfig()
)
@pytest.mark.parametrize(
"conflicting_option", ["enable_sleep_mode", "enable_cumem_allocator"]
)
def test_create_mem_pool_rejects_cumem_conflicts(conflicting_option):
model_config = _FakeModelConfig()
setattr(model_config, conflicting_option, True)
with pytest.raises(ValueError, match="custom_mem_pool is incompatible"):
worker.MooncakeStoreWorker._create_mem_pool(
{"custom_mem_pool": "NVLINK"}, model_config
)
@pytest.mark.parametrize(
("pool_type", "allocator_name"),
[("nvlink", "NVLinkAllocator"), ("barex", "BarexAllocator")],
)
def test_create_mem_pool_uses_mooncake_allocator(
monkeypatch, pool_type, allocator_name
):
allocator_cls = MagicMock()
allocator = allocator_cls.get_allocator.return_value
allocator_module = types.ModuleType("mooncake.allocator")
setattr(allocator_module, allocator_name, allocator_cls)
mooncake_module = types.ModuleType("mooncake")
mooncake_module.allocator = allocator_module # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, "mooncake", mooncake_module)
monkeypatch.setitem(sys.modules, "mooncake.allocator", allocator_module)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 2)
mem_pool = MagicMock()
mem_pool_ctor = MagicMock(return_value=mem_pool)
monkeypatch.setattr(torch.cuda, "MemPool", mem_pool_ctor)
result = worker.MooncakeStoreWorker._create_mem_pool(
{"custom_mem_pool": pool_type}, _FakeModelConfig()
)
assert result is mem_pool
allocator_cls.get_allocator.assert_called_once_with(torch.device("cuda", 2))
allocator.allocator.assert_called_once_with()
mem_pool_ctor.assert_called_once_with(allocator.allocator.return_value)
def test_pool_key_to_string_without_prefix_is_unchanged():
"""Default (empty) cache_prefix keeps keys byte-identical to the
historical unprefixed format so existing deployments keep their hits."""
@@ -68,6 +68,7 @@ class MockConnector(KVConnectorBase_V1):
# Override just build_kv_connector_stats
mock.build_kv_connector_stats = cls.build_kv_connector_stats
mock.get_kv_connector_stats.return_value = None
mock.get_mem_pool_context.return_value = None
return mock
@classmethod
@@ -201,6 +202,36 @@ def mc() -> MultiConnector:
return mc
def test_multi_connector_mem_pool_context_none(mc: MultiConnector):
assert mc.get_mem_pool_context() is None
def test_multi_connector_forwards_mem_pool_context(mc: MultiConnector):
context = MagicMock()
provider = MagicMock(spec_set=KVConnectorBase_V1)
provider.get_mem_pool_context.return_value = context
mc._connectors = [mc._connectors[0], provider]
assert mc.get_mem_pool_context() is context
provider.get_mem_pool_context.assert_called_once_with()
def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector):
providers = [
MagicMock(spec_set=KVConnectorBase_V1),
MagicMock(spec_set=KVConnectorBase_V1),
]
for provider in providers:
provider.get_mem_pool_context.return_value = MagicMock()
mc._connectors = providers
with pytest.raises(
ValueError,
match="Multiple connectors provide a KV cache memory pool",
):
mc.get_mem_pool_context()
# Helper function to compare directories recursively
def _compare_directories(dir1: Path, dir2: Path) -> bool:
"""Compares two directories recursively for identical content."""
@@ -320,6 +351,7 @@ def test_multi_example_connector_consistency():
# First three events are from initialization. Layer hooks run before the
# deferred load starts after the forward pass.
expected_worker_prefix = [
"get_mem_pool_context",
"register_kv_caches",
"set_host_xfer_buffer_ops",
"get_handshake_metadata",
@@ -7,6 +7,7 @@ import os
import queue
import tempfile
import textwrap
import threading
import time
import uuid
from collections import defaultdict
@@ -49,6 +50,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
NixlKVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
RemoteMeta,
ReqMeta,
compute_nixl_compatibility_hash,
)
from vllm.distributed.kv_transfer.kv_transfer_state import (
@@ -2087,6 +2090,10 @@ def recv_worker():
worker._recving_transfers = defaultdict(list)
worker._failed_recv_reqs = queue.Queue()
worker._recv_failures = set()
worker._handshake_lock = threading.RLock()
worker._handshake_futures = {}
worker._remote_agents = {}
worker._engine_by_address = {}
worker._replicated_pcp_done_sending = set()
worker._invalid_block_ids = queue.Queue()
worker._pending_recv_notifs = {}
@@ -2681,6 +2688,174 @@ def test_engine_ttl_disabled(default_vllm_config, dist_init):
assert engine_id in worker.dst_xfer_side_handles
@pytest.mark.cpu_test
class TestPeerReplacement:
@pytest.fixture(autouse=True)
def setup(self, monkeypatch):
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0})
config.kv_transfer_config.kv_buffer_device = "cpu"
platform = SimpleNamespace(
device_type="cpu",
discover_numa_topology=lambda: [],
get_nixl_memory_type=lambda: "DRAM",
is_rocm=lambda: False,
)
with (
patch.object(bw, "NixlWrapper", FakeNixlWrapper),
patch.object(bw, "current_platform", platform),
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
patch.object(
bw, "get_current_attn_backends", return_value=[FlashAttentionBackend]
),
):
self.worker = FakeNixlConnectorWorker(config, "local", hand_shake_latency=0)
self.nixl = self.worker.nixl_wrapper
self.transport = MagicMock(wraps=self.nixl)
self.worker.nixl_wrapper = self.transport
monkeypatch.setattr(
self.worker._handshake_initiation_executor,
"submit",
lambda *args: Future(),
)
self._connect("old")
yield
self.worker.shutdown()
def _connect(self, engine_id="new", host="localhost", port=1234):
future = self.worker._ensure_handshake(engine_id, host, port, 1)
self.worker.REMOTE_ENGINE_ID = engine_id
self.nixl.REMOTE_AGENT_NAME = engine_id
future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id))
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True):
metadata = NixlConnectorMetadata()
metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta(
local_block_ids=(blocks,),
local_physical_block_ids=(blocks,),
tp_size=1,
remote=RemoteMeta(
([2],), "localhost", 1234, engine_id, f"{engine_id}-prefill"
),
awaiting_kvs=awaiting_kvs,
)
return metadata
def test_cleans_old_peer_after_confirmed_handshake(self):
self._connect("healthy", host="other-host")
old_handle = self.worker.dst_xfer_side_handles["old"][0]
self.worker.start_load_kv(self._request())
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self._connect()
self.worker.get_transfer_results()
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
self.transport.remove_remote_agent.assert_called_once_with("old")
assert set(self.worker._remote_agents) == {"new", "healthy"}
assert self.worker._engine_by_address == {
("localhost", 1234): "new",
("other-host", 1234): "healthy",
}
with pytest.raises(KeyError):
self.worker.transfer_topo.get_engine_info("old")
self.worker.start_load_kv(NixlConnectorMetadata())
assert self.worker.get_transfer_results().finished_recving == {"new-req"}
self.transport.remove_remote_agent.assert_called_once_with("old")
@pytest.mark.parametrize("host,port", [("other-host", 1234), ("localhost", 5678)])
def test_preserves_other_addresses(self, host, port):
self._connect(host=host, port=port)
self.worker.get_transfer_results()
assert "old" in self.worker._remote_agents
self.transport.remove_remote_agent.assert_not_called()
def test_preserves_old_peer_when_handshake_fails(self):
self.worker.start_load_kv(self._request())
self.worker._handshake_futures["new"].set_exception(
RuntimeError("handshake failed")
)
assert self.worker.get_transfer_results().failed_recving == {"new-req"}
assert "old" in self.worker._remote_agents
assert self.worker._engine_by_address == {("localhost", 1234): "old"}
self.transport.remove_remote_agent.assert_not_called()
@pytest.mark.parametrize(
"states,failed",
[
(["PROC", "DONE"], False),
(["ERR", "PROC", "DONE"], True),
(["ERR", "PROC", "ERR"], True),
],
ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"],
)
def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed):
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1]
self.transport.check_xfer_state.side_effect = states
self._connect()
result = self.worker.get_transfer_results()
assert result.finished_recving == result.failed_recving == set()
assert self.worker.get_block_ids_with_load_errors() == set()
self.transport.remove_remote_agent.assert_not_called()
result = self.worker.get_transfer_results()
assert result.finished_recving == {"old-req"}
assert result.failed_recving == ({"old-req"} if failed else set())
assert self.worker.get_block_ids_with_load_errors() == (
{1} if failed else set()
)
assert self.transport.release_xfer_handle.call_count == (2 if failed else 1)
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_waits_for_queued_notification(self):
metadata = self._request("old", blocks=())
self.worker._recving_metadata.update(metadata.reqs_to_recv)
self.worker._background_nixl_handshake(
"old-req", "old", metadata.reqs_to_recv["old-req"]
)
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self.worker.start_load_kv(NixlConnectorMetadata())
self.transport.send_notif.assert_called_once_with(
"old", notif_msg=b"old-prefill:1"
)
result = self.worker.get_transfer_results()
assert result.finished_recving == {"old-req"}
assert result.failed_recving == set()
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_waits_for_other_handshakes(self):
self.worker._ensure_handshake("other", "other-host", 1234, 1)
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self._connect("other", host="other-host")
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_rehandshakes_queued_request_for_released_engine(self):
# A request queued after its engine's handshake, but drained only after
# that engine was replaced and released, must not read from it.
metadata = self._request("old", blocks=(), awaiting_kvs=False)
self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"]))
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
self.worker.start_load_kv(NixlConnectorMetadata())
assert "old" in self.worker._handshake_futures
self.transport.send_notif.assert_not_called()
def test_transfer_topology_unregister():
"""TransferTopology.unregister_remote_engine removes the engine."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
@@ -4,6 +4,7 @@
import gc
import queue
import threading
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
@@ -54,6 +55,9 @@ def region_pull_worker():
worker.enable_permute_local_kv = False
worker.enable_heterogeneous_attn_post_process = False
worker._engine_last_active = {}
worker._handshake_lock = threading.RLock()
worker._handshake_futures = {}
worker._transfer_layer_group_ids = ()
worker._bidirectional_kv_xfer_enabled = False
worker._recving_transfers = {}
worker.use_mla, worker._has_mamba = True, False
@@ -94,6 +98,7 @@ def region_pull_worker():
worker.dst_num_blocks = {"P": 100, "D": 100}
worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]}
worker._remote_agents = {"P": {(0, 0): "P-rank0"}}
worker._engine_by_address = {("localhost", 1): "P"}
worker._read_blocks_mixed = MagicMock()
worker.nixl_wrapper = MagicMock()
return worker
@@ -395,6 +395,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w._group_spec_types = (FullAttentionSpec,)
w._engine_ttl = 0.0
w._engine_last_active = {}
w._engine_by_address = {}
# Track _do_start_push_kv invocations.
calls: list[tuple[str, Any, dict[str, Any]]] = []
@@ -0,0 +1,466 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for SimpleCPUOffloadConnector metrics."""
from __future__ import annotations
import dataclasses
from types import SimpleNamespace
from typing import Any
import pytest
from prometheus_client import Counter, Gauge, Histogram
from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import (
_make_connector,
)
from tests.v1.simple_kv_offload.test_scheduler import (
make_request,
make_scheduler,
make_scheduler_output,
)
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
SimpleCPUOffloadConnector,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.simple_kv_offload.manager import (
BoundaryStoreStats,
LoadRequestState,
TransferMeta,
)
from vllm.v1.simple_kv_offload.metrics import (
INFO_LABELS,
OUTCOME_TO_FIELD,
MetricName,
SimpleCPUOffloadStats,
_MetricType,
_StatsKey,
)
SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES
LOAD_BLOCKS = MetricName.LOAD_BLOCKS
USED_BLOCKS = MetricName.USED_BLOCKS
PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS
INFO = MetricName.INFO
class _FakeMetric:
def __init__(self, **kwargs: Any):
self.kwargs = kwargs
self.children: list[_FakeMetric] = []
self.observed: list[int | float] = []
self.increments: list[int | float] = []
self.set_values: list[int | float] = []
self.labelvalues: tuple[object, ...] = ()
def labels(self, *labelvalues: object) -> _FakeMetric:
child = _FakeMetric(**self.kwargs)
child.labelvalues = labelvalues
self.children.append(child)
return child
def observe(self, value: int | float) -> None:
self.observed.append(value)
def inc(self, value: int | float) -> None:
self.increments.append(value)
def set(self, value: int | float) -> None:
self.set_values.append(value)
def _fake_metric_types() -> dict[type, type]:
return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric}
def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any:
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
return SimpleCPUOffloadPromMetrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
metric_types=_fake_metric_types(),
labelnames=["model_name", "engine"],
per_engine_labelvalues=per_engine or {0: ["model", "0"]},
)
def _stats_payload(types: dict, data: dict) -> dict[str, Any]:
return {_StatsKey.TYPES: types, _StatsKey.DATA: data}
def _values(stats: SimpleCPUOffloadStats, name: str) -> dict:
return stats.data[_StatsKey.DATA][name]
# ---------------------------------------------------------------------------
# Stats class
# ---------------------------------------------------------------------------
def test_build_kv_connector_stats_none_and_empty() -> None:
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None)
assert isinstance(stats, SimpleCPUOffloadStats)
assert stats.is_empty()
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={})
assert isinstance(stats, SimpleCPUOffloadStats)
assert stats.is_empty()
def test_stats_round_trip_through_serialized_dict() -> None:
stats = SimpleCPUOffloadStats()
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats.increase_counter(LOAD_BLOCKS, 4)
stats.set_gauge(USED_BLOCKS, 7)
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict())
assert isinstance(rebuilt, SimpleCPUOffloadStats)
assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3}
assert _values(rebuilt, LOAD_BLOCKS) == {(): 4}
assert _values(rebuilt, USED_BLOCKS) == {(): 7}
assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1}
def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None:
stats1 = SimpleCPUOffloadStats()
stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",))
stats1.set_gauge(USED_BLOCKS, 7)
stats2 = SimpleCPUOffloadStats()
stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",))
stats2.set_gauge(USED_BLOCKS, 2)
result = stats1.aggregate(stats2)
assert result is stats1
outcomes = _values(stats1, SAVE_OUTCOMES)
assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1}
assert _values(stats1, USED_BLOCKS) == {(): 2}
def test_stats_aggregate_merges_types_from_other() -> None:
stats1 = SimpleCPUOffloadStats()
stats1.set_gauge(USED_BLOCKS, 1)
stats2 = SimpleCPUOffloadStats()
stats2.increase_counter(LOAD_BLOCKS, 2)
stats1.aggregate(stats2)
assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER
assert _values(stats1, LOAD_BLOCKS) == {(): 2}
def test_stats_reduce_and_reset() -> None:
stats = SimpleCPUOffloadStats()
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats.set_gauge(PENDING_STORE_BLOCKS, 4)
stats.increase_counter(LOAD_BLOCKS, 9)
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
reduced = stats.reduce()
assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3
assert reduced[PENDING_STORE_BLOCKS] == 4
assert reduced[LOAD_BLOCKS] == 9
assert not any(key.startswith(INFO) for key in reduced)
stats.reset()
assert stats.is_empty()
def test_outcome_labels_match_boundary_store_stats_fields() -> None:
"""Every outcome label maps 1:1 to a BoundaryStoreStats counter field."""
boundary_fields = {
f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published"
}
assert set(OUTCOME_TO_FIELD.values()) == boundary_fields
assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD)
# ---------------------------------------------------------------------------
# Prom metrics class
# ---------------------------------------------------------------------------
def test_prom_metrics_registers_tier1_metrics() -> None:
prom = _make_prom_metrics()
defs = prom._defs
assert set(defs) == {
SAVE_OUTCOMES,
LOAD_BLOCKS,
USED_BLOCKS,
PENDING_STORE_BLOCKS,
INFO,
}
# The prom class registers literal names (required by the docs
# generator); they must stay in sync with the emission-side constants.
assert {m.kwargs["name"] for m in defs.values()} == {
SAVE_OUTCOMES,
LOAD_BLOCKS,
USED_BLOCKS,
PENDING_STORE_BLOCKS,
INFO,
}
for metric in defs.values():
assert metric.kwargs["documentation"]
assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"]
assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [
"model_name",
"engine",
"outcome",
]
assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
assert defs[INFO].kwargs["labelnames"] == [
"model_name",
"engine",
*INFO_LABELS,
]
assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
# mostrecent: multiprocess scrapes expose one series per label set, not
# one pid-tagged series per writer process.
for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO):
assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent"
def test_prom_metrics_observe_routes_each_series() -> None:
prom = _make_prom_metrics()
prom.observe(
_stats_payload(
types={
SAVE_OUTCOMES: _MetricType.COUNTER,
LOAD_BLOCKS: _MetricType.COUNTER,
USED_BLOCKS: _MetricType.GAUGE,
PENDING_STORE_BLOCKS: _MetricType.GAUGE,
INFO: _MetricType.GAUGE,
},
data={
SAVE_OUTCOMES: {("stored",): 3},
LOAD_BLOCKS: {(): 4},
USED_BLOCKS: {(): 5},
PENDING_STORE_BLOCKS: {(): 1},
INFO: {("cpu", "false", "false", "64"): 1},
},
)
)
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3]
assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4]
assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5]
assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1]
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1]
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == (
"model",
"0",
"stored",
)
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == (
"model",
"0",
"cpu",
"false",
"false",
"64",
)
def test_prom_metrics_reuses_bound_children() -> None:
prom = _make_prom_metrics()
payload = _stats_payload(
types={SAVE_OUTCOMES: _MetricType.COUNTER},
data={SAVE_OUTCOMES: {("stored",): 1}},
)
prom.observe(payload)
prom.observe(payload)
child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))]
assert child.increments == [1, 1]
assert len(prom._defs[SAVE_OUTCOMES].children) == 1
def test_prom_metrics_rejects_unknown_metric() -> None:
prom = _make_prom_metrics()
with pytest.raises(AssertionError, match="Unknown"):
prom.observe(
_stats_payload(
types={"vllm:not_a_metric": _MetricType.COUNTER},
data={"vllm:not_a_metric": {(): 1}},
)
)
def test_prom_metrics_rejects_wrong_label_count() -> None:
prom = _make_prom_metrics()
with pytest.raises(AssertionError, match="labels"):
prom.observe(
_stats_payload(
types={SAVE_OUTCOMES: _MetricType.COUNTER},
data={SAVE_OUTCOMES: {("stored", "extra"): 1}},
)
)
def test_prom_metrics_routes_per_engine() -> None:
prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]})
prom.observe(
_stats_payload(
types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}}
),
engine_idx=1,
)
assert (0, USED_BLOCKS, ()) not in prom._metrics
assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3]
def test_connector_build_prom_metrics() -> None:
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
prom = SimpleCPUOffloadConnector.build_prom_metrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
metric_types=_fake_metric_types(),
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
assert isinstance(prom, SimpleCPUOffloadPromMetrics)
# ---------------------------------------------------------------------------
# Manager: interval stats
# ---------------------------------------------------------------------------
def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None:
fixture = make_scheduler()
sched = fixture.scheduler
sched.boundary_store_stats.stored += 3
sched.boundary_store_stats.dropped_cpu_full += 2
sched.boundary_store_stats.skipped_in_flight += 1
stats = sched.get_stats()
assert _values(stats, SAVE_OUTCOMES) == {
("stored",): 3,
("dropped_cpu_full",): 2,
("skipped_in_flight",): 1,
}
# Deltas are per-interval: a second drain reports no new outcomes.
stats2 = sched.get_stats()
assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA]
def test_manager_reset_preserves_undrained_outcome_deltas() -> None:
fixture = make_scheduler()
sched = fixture.scheduler
sched.boundary_store_stats.stored += 3
sched.reset()
stats = sched.get_stats()
assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3}
def test_manager_get_stats_gauges_and_info_labels() -> None:
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
sched._store_event_to_blocks[0] = TransferMeta([10], [11])
sched._pending_finished_stores.append(TransferMeta([12], [13]))
sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15])
stats = sched.get_stats()
assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3}
used = _values(stats, USED_BLOCKS)[()]
assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks()
info = _values(stats, INFO)
assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1}
# Gauges are re-reported every drain.
stats2 = sched.get_stats()
assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3}
def test_manager_info_labels_reflect_disk_mode() -> None:
connector = _make_connector(
extra_config={
"kv_offload_backend": "disk",
"disk_path": "/tmp/fake-disk-metrics",
"disk_capacity_bytes": 1024**3,
"use_page_cache": True,
}
)
sched = connector.scheduler_manager
assert sched is not None
assert sched._info_labelvalues[0] == "disk"
assert sched._info_labelvalues[1] == "true"
stats = sched.get_stats()
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None:
"""CPU backend ignores use_page_cache; the info gauge must not claim it."""
connector = _make_connector(extra_config={"use_page_cache": True})
sched = connector.scheduler_manager
assert sched is not None
assert sched._info_labelvalues[:2] == ("cpu", "false")
stats = sched.get_stats()
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
def test_manager_counts_load_blocks_completed() -> None:
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
request = make_request(num_blocks=2)
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
sched._reqs_to_load[request.request_id] = LoadRequestState(
request=request,
transfer_meta=TransferMeta(
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
),
)
sched.build_connector_meta(make_scheduler_output({}))
stats = sched.get_stats()
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
sched.update_connector_output(
KVConnectorOutput(finished_recving={request.request_id})
)
stats = sched.get_stats()
assert _values(stats, LOAD_BLOCKS) == {(): 2}
def test_manager_abandoned_load_counts_as_completed_after_reset() -> None:
"""Loads abandoned by reset() that still finish on the worker are
counted as loaded blocks.
"""
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
request = make_request(num_blocks=2)
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
sched._reqs_to_load[request.request_id] = LoadRequestState(
request=request,
transfer_meta=TransferMeta(
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
),
)
sched.build_connector_meta(make_scheduler_output({}))
stats = sched.get_stats()
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
assert sched.reset() is False
assert set(sched._abandoned_reqs_to_load) == {request.request_id}
sched.update_connector_output(
KVConnectorOutput(finished_recving={request.request_id})
)
assert sched._abandoned_reqs_to_load == {}
stats = sched.get_stats()
assert _values(stats, LOAD_BLOCKS) == {(): 2}
+2 -2
View File
@@ -2316,8 +2316,8 @@ def _make_hybrid_attention_mamba_scheduler(
def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
"""Replicated groups must not be scaled by the DCP world size.
``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP
size and every other spec 1. Reconstructing a group's geometry as
The spec's ``dcp_sharded`` flag controls its effective block size.
Reconstructing a group's geometry as
``spec.block_size * cp_world_size`` over-scales the replicated groups, so
the eager store scan believes far fewer of their blocks are ready and
silently offloads only a fraction of them.
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import patch
import numpy as np
import pytest
@@ -9,7 +10,7 @@ import torch
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
prepare_dflash_inputs,
DFlashSpeculator,
)
pytestmark = pytest.mark.skipif(
@@ -24,6 +25,7 @@ def _run_prepare(
cp_rank: int = 0,
cp_size: int = 1,
cp_interleave: int = 1,
draft_dcp_size: int | None = None,
):
device = torch.device("cuda")
max_num_reqs = 4
@@ -40,7 +42,9 @@ def _run_prepare(
)
input_batch = SimpleNamespace(
num_reqs=1,
num_tokens=4,
num_scheduled_tokens=np.array([4], dtype=np.int32),
seq_lens_cpu_upper_bound=torch.tensor([target_positions[-1] + 1]),
positions=torch.tensor(target_positions, dtype=torch.int64, device=device),
query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device),
idx_mapping=torch.tensor([2], dtype=torch.int32, device=device),
@@ -74,36 +78,65 @@ def _run_prepare(
next_prefill_tokens = torch.zeros_like(last_sampled)
block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device)
prepare_dflash_inputs(
input_buffers,
query_slot_mapping,
context_positions,
context_slot_mapping,
sample_indices,
sample_pos,
sample_idx_mapping,
temperature,
seeds,
input_batch,
torch.tensor([1], dtype=torch.int32, device=device),
torch.tensor([2], dtype=torch.int32, device=device),
last_sampled,
next_prefill_tokens,
input_temperature,
input_seeds,
block_table,
4,
cp_rank,
cp_size,
cp_interleave,
123,
num_speculative_steps,
num_speculative_steps,
max_num_reqs,
max_num_tokens,
128,
class InputsPrepared(Exception):
pass
draft = SimpleNamespace(
input_buffers=input_buffers,
context_positions=context_positions,
_context_slot_mappings=context_slot_mapping.unsqueeze(0),
sample_indices=sample_indices,
sample_pos=sample_pos,
sample_idx_mapping=sample_idx_mapping,
temperature=temperature,
seeds=seeds,
hidden_states=torch.zeros(4, 1, device=device),
prepare_context_anchor=lambda *args: None,
query_cudagraph_manager=None,
dp_size=1,
dp_rank=0,
pcp_manager=None,
draft_kv_cache_group_id=0,
draft_kv_cache_group_ids=[0],
_layer_group_idx=None,
block_tables=SimpleNamespace(
slot_mappings=query_slot_mapping.unsqueeze(0),
input_block_tables=[block_table],
kernel_block_sizes=[4],
cp_rank=cp_rank,
cp_size=cp_size,
cp_interleave=cp_interleave,
),
dcp_size=cp_size if draft_dcp_size is None else draft_dcp_size,
parallel_drafting_token_id=123,
num_query_per_req=num_speculative_steps,
num_speculative_steps=num_speculative_steps,
max_num_reqs=max_num_reqs,
max_num_tokens=max_num_tokens,
max_model_len=128,
sample_from_anchor=True,
)
with (
patch(
"vllm.v1.worker.gpu.spec_decode.dflash.speculator.dispatch_cg_and_sync_dp",
side_effect=InputsPrepared,
),
pytest.raises(InputsPrepared),
):
DFlashSpeculator.propose(
draft,
input_batch,
{},
{},
draft.hidden_states,
None,
torch.tensor([1], dtype=torch.int32, device=device),
torch.tensor([2], dtype=torch.int32, device=device),
last_sampled,
next_prefill_tokens,
input_temperature,
input_seeds,
)
torch.accelerator.synchronize()
return SimpleNamespace(
input_buffers=input_buffers,
@@ -118,12 +151,17 @@ def _run_prepare(
)
def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
@pytest.mark.parametrize("cp_rank,cp_size", [(0, 1), (0, 2), (1, 2)])
def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size):
# Positions 10/11 use physical block 7. Rejected positions 12/13 would use
# block 8, but must be PAD context rather than contaminating draft KV.
out = _run_prepare(
target_positions=[10, 11, 12, 13],
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
cp_rank=cp_rank,
cp_size=cp_size,
cp_interleave=2,
draft_dcp_size=1,
)
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
@@ -17,6 +17,7 @@ from types import MethodType, SimpleNamespace
from unittest.mock import patch
import numpy as np
import pytest
import torch
from vllm.config.compilation import CUDAGraphMode
@@ -53,6 +54,7 @@ def _make_fake_speculator(
arange_np=np.arange(max_num_reqs + 1, dtype=np.int32),
draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool),
block_tables=fake_block_tables,
dcp_size=1,
input_buffers=fake_input_buffers,
attn_groups=[],
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
@@ -152,14 +154,17 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len():
assert torch.equal(bound, torch.tensor([1024, 503], dtype=torch.int32))
def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
@pytest.mark.parametrize("draft_dcp_size", [1, 2])
def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(draft_dcp_size):
fake = _make_fake_speculator()
fake.dcp_size = draft_dcp_size
fake.block_tables.cp_size = 2
fake.block_tables.cp_rank = 1
fake.block_tables.cp_interleave = 4
fake.input_buffers.seq_lens[:3] = torch.tensor([5, 9, 16])
def fake_prepare(out, seq_lens, num_reqs, dcp_size, dcp_rank, cp_interleave):
assert draft_dcp_size > 1
assert seq_lens is fake.input_buffers.seq_lens
assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (3, 2, 1, 4)
out[:num_reqs].copy_(torch.tensor([1, 4, 8], dtype=torch.int32))
@@ -176,6 +181,9 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
)
local = captured["dcp_local_seq_lens"]
if draft_dcp_size == 1:
assert local is None
return
assert isinstance(local, torch.Tensor)
assert local.data_ptr() == fake.input_buffers.dcp_local_seq_lens.data_ptr()
assert local.tolist() == [1, 4, 8, 0]
+34
View File
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
@@ -7,6 +9,38 @@ from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.v1.worker.cp_utils import should_skip_dcp_context_attention
def test_cp_checks_allow_a_replicated_draft_backend_without_dcp(monkeypatch):
from vllm.v1.worker import cp_utils
config = SimpleNamespace(
parallel_config=SimpleNamespace(
prefill_context_parallel_size=1,
decode_context_parallel_size=4,
cp_kv_cache_interleave_size=16,
),
speculative_config=object(),
)
target = SimpleNamespace(
impl=SimpleNamespace(
dcp_world_size=4,
supports_mtp_with_cp_non_trivial_interleave_size=True,
need_to_return_lse_for_decode=True,
)
)
draft = SimpleNamespace(
impl=SimpleNamespace(
dcp_world_size=1,
supports_mtp_with_cp_non_trivial_interleave_size=False,
need_to_return_lse_for_decode=False,
)
)
layers = {"target": target, "draft": draft}
monkeypatch.setattr(cp_utils, "get_layers_from_vllm_config", lambda *args: layers)
cp_utils.check_attention_cp_compatibility(config, target_layer_names={"target"})
with pytest.raises(AssertionError, match="MTP with cp_kv_cache_interleave_size"):
cp_utils.check_attention_cp_compatibility(config)
def test_skip_gate_only_for_zero_context():
assert should_skip_dcp_context_attention(torch.zeros(3, dtype=torch.int32))
assert not should_skip_dcp_context_attention(
+92
View File
@@ -2,8 +2,17 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from types import SimpleNamespace
import pytest
import torch
from vllm.config import ParallelConfig
from vllm.v1.kv_cache_interface import FullAttentionSpec
from vllm.v1.worker.gpu import model_runner
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_config
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
@dataclass
@@ -49,3 +58,86 @@ def test_dspark_parallel_config_disables_eplb_atomically():
assert draft_config.eplb_config.num_redundant_experts == 0
assert not draft_config.enable_elastic_ep
assert draft_config.eplb_config is not target_config.eplb_config
@pytest.mark.parametrize("pcp_size", [1, 4])
@pytest.mark.parametrize("dcp_size", [1, 4])
@pytest.mark.parametrize("use_mla", [False, True])
def test_draft_context_parallelism_without_changing_target(
monkeypatch, pcp_size, dcp_size, use_mla
):
target_parallel = ParallelConfig(
tensor_parallel_size=4,
prefill_context_parallel_size=pcp_size,
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=16,
distributed_executor_backend="mp",
)
target_config = SimpleNamespace(
parallel_config=target_parallel,
speculative_config=SimpleNamespace(
draft_model_config=SimpleNamespace(use_mla=use_mla)
),
)
class CapturedConfig(Exception):
pass
def capture_init(self, config, device):
raise CapturedConfig(config)
monkeypatch.setattr(DraftModelSpeculator, "__init__", capture_init)
with pytest.raises(CapturedConfig) as captured:
DFlashSpeculator(target_config, device=None)
draft_parallel = captured.value.args[0].parallel_config
assert draft_parallel.tensor_parallel_size == 4
assert draft_parallel.prefill_context_parallel_size == 1
assert draft_parallel.decode_context_parallel_size == (dcp_size if use_mla else 1)
assert draft_parallel.cp_kv_cache_interleave_size == 16
assert target_parallel.prefill_context_parallel_size == pcp_size
assert target_parallel.decode_context_parallel_size == dcp_size
assert target_parallel.cp_kv_cache_interleave_size == 16
@pytest.mark.parametrize("configured_dcp", [1, 4])
def test_attention_uses_draft_dcp_setting_inside_target_process_group(
monkeypatch, configured_dcp
):
import vllm.config as config_module
from vllm.distributed import parallel_state
from vllm.v1.attention.backend import AttentionImplBase
config = SimpleNamespace(
parallel_config=SimpleNamespace(decode_context_parallel_size=configured_dcp)
)
monkeypatch.setattr(
config_module, "get_current_vllm_config_or_none", lambda: config
)
monkeypatch.setattr(
parallel_state, "_DCP", SimpleNamespace(world_size=4, rank_in_group=2)
)
impl = AttentionImplBase()
assert impl.dcp_world_size == configured_dcp
assert impl.dcp_rank == (0 if configured_dcp == 1 else 2)
@pytest.mark.parametrize("draft_dcp_size", [1, 4])
def test_runner_marks_only_replicated_draft_caches(monkeypatch, draft_dcp_size):
spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.bfloat16
)
monkeypatch.setattr(
model_runner, "get_kv_cache_spec", lambda _: {"target": spec, "draft": spec}
)
runner = object.__new__(model_runner.GPUModelRunner)
runner.vllm_config = None
runner.dcp_size = 4
runner.speculator = object.__new__(DFlashSpeculator)
runner.speculator.dcp_size = draft_dcp_size
runner.speculator.draft_attn_layer_names = {"draft"}
specs = runner.get_kv_cache_spec()
assert specs["target"].dcp_sharded
assert specs["draft"].dcp_sharded == (draft_dcp_size == 4)
@@ -88,6 +88,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32),
seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32),
)
speculator.dcp_size = 1
speculator.block_tables = SimpleNamespace(
cp_size=1,
input_block_tables=[torch.zeros(4, 1, dtype=torch.int32)],
@@ -183,6 +184,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected)
parallel_config=SimpleNamespace(
data_parallel_size=1,
data_parallel_rank=0,
decode_context_parallel_size=1,
),
)
+9 -7
View File
@@ -171,22 +171,23 @@ def test_block_tables_skip_custom_slot_mapping_groups():
@pytest.mark.parametrize("cp_rank", range(4))
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
"""DCP interleave is expressed in logical-block token coordinates."""
"""Only sharded groups use DCP interleave in logical-block coordinates."""
device = torch.device("cuda")
block_tables = BlockTables(
block_sizes=[128],
block_sizes=[128, 128],
max_num_reqs=1,
max_num_batched_tokens=1024,
max_num_blocks_per_group=[2],
max_num_blocks_per_group=[2, 8],
device=device,
kernel_block_sizes=[64],
kernel_block_sizes=[64, 64],
dcp_sharded=[True, False],
cp_size=4,
cp_rank=cp_rank,
cp_interleave=128,
)
block_tables.append_block_ids(
req_index=0,
new_block_ids=([5, 9],),
new_block_ids=([5, 9], list(range(10, 18))),
overwrite=True,
)
block_tables.apply_staged_writes()
@@ -199,7 +200,7 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
query_start_loc,
positions,
num_tokens_padded=1024,
)[0]
)
expected = torch.full((1024,), -1, dtype=torch.int64, device=device)
first_start = cp_rank * 128
@@ -210,7 +211,8 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
expected[second_start : second_start + 128] = torch.arange(
9 * 128, 10 * 128, dtype=torch.int64, device=device
)
assert torch.equal(actual, expected)
assert torch.equal(actual[0], expected)
assert torch.equal(actual[1], positions + 10 * 128)
def test_v1_block_table_move_row_clears_vacated_row():
+56
View File
@@ -372,6 +372,62 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
_compare_objs(input_batch, ref_input_batch)
@pytest.mark.parametrize("device", DEVICES)
def test_condense_clears_stale_allowed_token_ids_mask(device: str):
"""condense() must clear the mask row a constrained request is moved out
of. Otherwise a later request that reuses that row without
allowed_token_ids inherits the stale whitelist.
See https://github.com/vllm-project/vllm/issues/43894.
"""
batch_size = 4
allowed_token_id = 13
input_batch = InputBatch(
max_num_reqs=batch_size,
max_model_len=1024,
max_num_batched_tokens=1024,
device=torch.device(device),
vocab_size=VOCAB_SIZE,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
def _make_req(suffix: int, allowed_token_ids=None) -> CachedRequestState:
return CachedRequestState(
req_id=f"req_id_{suffix}",
prompt_token_ids=[1, 2, 3],
sampling_params=SamplingParams(allowed_token_ids=allowed_token_ids),
pooling_params=None,
mm_features=[],
block_ids=([],),
generator=None,
num_computed_tokens=0,
output_token_ids=[],
)
# Only req_id_2 is constrained, and it lands on the highest row.
assert input_batch.add_request(_make_req(0)) == 0
assert input_batch.add_request(_make_req(1)) == 1
assert input_batch.add_request(_make_req(2, [allowed_token_id])) == 2
mask = input_batch.allowed_token_ids_mask_cpu_tensor
assert mask is not None
# The constrained row masks every token except the single allowed id.
assert not mask[2][allowed_token_id].item()
assert int(mask[2].sum().item()) == VOCAB_SIZE - 1
# Free row 0, then condense: req_id_2 slides from row 2 down into row 0.
input_batch.remove_request("req_id_0")
input_batch.condense()
assert input_batch.req_id_to_index["req_id_2"] == 0
assert int(mask[2].sum().item()) == 0
# A new unrestricted request reuses row 2 and must stay unconstrained.
assert input_batch.add_request(_make_req(3)) == 2
assert "req_id_3" not in input_batch.has_allowed_token_ids
assert int(mask[2].sum().item()) == 0
@pytest.mark.parametrize("device", DEVICES)
def test_swap_states_preserves_allowed_token_ids_mask(device: str):
"""swap_states() must exchange the two allowed_token_ids_mask rows.
+18
View File
@@ -75,6 +75,24 @@ def test_validate_config_rejects_full_graph_for_prefills():
)
@pytest.mark.parametrize("draft_uses_mla", [False, True])
@pytest.mark.parametrize("dcp_size", [1, 2, 4])
def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size):
config = _make_config(CUDAGraphMode.NONE)
config.parallel_config.prefill_context_parallel_size = 4
config.parallel_config.decode_context_parallel_size = dcp_size
config.model_config.hf_text_config.index_topk = 1
config.speculative_config = SimpleNamespace(
use_dspark=lambda: True,
draft_model_config=SimpleNamespace(use_mla=draft_uses_mla),
)
if draft_uses_mla and dcp_size == 2:
with pytest.raises(NotImplementedError, match="requires DCP=1 or DCP=PCP"):
PCPManager.validate_config(config, supports_mm_inputs=False)
return
PCPManager.validate_config(config, supports_mm_inputs=False)
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
manager = PCPManager(
pcp_world_size=2,
+51 -23
View File
@@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail(
_OPS_REGISTERED = False
_AITER_SITUV2_ACT_ENV = {
"a8w4": "AITER_SITUV2_A8W4",
"a4w4": "AITER_SITUV2_A4W4",
}
def _resolve_situv2_activation() -> str:
"""VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4.
auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4.
"""
value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower()
if value in ("auto", "1"):
return "a4w4"
if value == "0":
return "a16w4"
return value
def _sync_aiter_situv2_moe_env() -> None:
"""Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env.
"""Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2.
AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 /
AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first).
When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4
(afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override.
Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4
dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs
afp4 FlyDSL but falls back to heuristic configs, which is not the
tuned a4w4 path this recipe is meant to use.
AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4,
so exactly one is set for a8w4/a4w4 and both are cleared for a16w4.
"""
import os
import vllm.envs as envs
if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2:
os.environ["AITER_SITUV2_A4W4"] = "1"
os.environ.pop("AITER_SITUV2_A8W4", None)
else:
os.environ.pop("AITER_SITUV2_A4W4", None)
selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation())
for name in _AITER_SITUV2_ACT_ENV.values():
if name == selected:
os.environ[name] = "1"
else:
os.environ.pop(name, None)
class rocm_aiter_ops:
@@ -1917,7 +1928,8 @@ class rocm_aiter_ops:
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4).
VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation
dtype (a16w4 | a8w4 | a4w4).
VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM.
Note:
@@ -1987,7 +1999,7 @@ class rocm_aiter_ops:
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
_MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
_MOE_SITUV2 = _resolve_situv2_activation()
# TODO: Consolidate under _LINEAR_ENABLED
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
# Lazily probed: whether aiter.topk_softmax supports the
@@ -2016,7 +2028,7 @@ class rocm_aiter_ops:
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
cls._MOE_SITUV2 = _resolve_situv2_activation()
_sync_aiter_situv2_moe_env()
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
@@ -2132,9 +2144,25 @@ class rocm_aiter_ops:
@classmethod
@if_aiter_supported
def is_fused_moe_situv2_enabled(cls) -> bool:
# _MOE_SITUV2 is a variant of aiter fused moe, so aiter
# fused moe must be enabled as well.
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2
"""True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected."""
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4"
@classmethod
@if_aiter_supported
def get_fused_moe_situv2_activation(cls) -> str:
"""SiTUv2 activation dtype name: a16w4, a8w4 or a4w4."""
return cls._MOE_SITUV2
@classmethod
@if_aiter_supported
def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool:
"""Whether the SiTUv2 kernels expect gate/up-interleaved w13.
The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and
a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode
(experts/rocm_aiter_moe.py) must both use this.
"""
return cls._MOE_SITUV2 == "a8w4"
@classmethod
@if_aiter_supported
+20
View File
@@ -1725,6 +1725,26 @@ class VllmConfig:
)
self.compilation_config.mode = CompilationMode.NONE
# TODO: This is a stopgap that turns sequence parallelism / async TP
# off under batch invariance. Make the sequence-parallel reduce-scatter
# path batch-invariant instead so they can stay enabled (#56370).
# Sequence parallelism / async TP rewrite all_reduce + rms_norm into a
# reduce-scatter whose reduction order depends on the batch, so they
# are not batch-invariant. Decide this before the breakable-CUDA-graph
# default below, which declines to auto-enable when they are on.
pass_config = self.compilation_config.pass_config
if envs.VLLM_BATCH_INVARIANT and (
pass_config.enable_sp or pass_config.fuse_gemm_comms
):
logger.warning_once(
"Disabling sequence parallelism and async TP "
"(pass_config.enable_sp / fuse_gemm_comms) when "
"VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path "
"is not batch-invariant (see vllm-project/vllm#56370)."
)
pass_config.enable_sp = False
pass_config.fuse_gemm_comms = False
breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph()
if not breakable_cudagraph_enabled and (
@@ -42,6 +42,7 @@ The class provides the following primitives:
import enum
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
from contextlib import AbstractContextManager
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Literal
@@ -227,6 +228,13 @@ class KVConnectorBase_V1(ABC):
# Worker-side methods
# ==============================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a custom KV cache allocation context, if configured.
Returning None uses the engine's default memory pool.
"""
return None
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
"""Set the connector metadata from the scheduler.
@@ -59,7 +59,6 @@ from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import (
dcp_world_size_for_kv_cache_spec,
resolve_dcp_kv_block_size,
)
from vllm.v1.kv_cache_interface import (
@@ -259,13 +258,7 @@ class DecodeBenchConnectorScheduler:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
self.kv_cache_groups = kv_cache_config.kv_cache_groups
self.group_block_sizes = tuple(
resolve_dcp_kv_block_size(
group.kv_cache_spec,
dcp_world_size_for_kv_cache_spec(
group.kv_cache_spec,
dcp_world_size,
),
)
resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_world_size)
for group in self.kv_cache_groups
)
@@ -13,6 +13,7 @@ enabling prefix caching via hash-based deduplication.
from collections import Counter
from collections.abc import Iterable, Sequence
from contextlib import AbstractContextManager
from typing import Any
import torch
@@ -322,6 +323,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
# Worker-side methods
# ============================================================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a context manager for the custom MemPool, or None.
Called by the Worker before ``initialize_kv_cache`` so that KV
cache is allocated from the Mooncake-managed pool when
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
"""
if self.connector_worker is None:
return None
return self.connector_worker.get_mem_pool_context()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
@@ -19,13 +19,14 @@ import threading
import time
from collections.abc import Callable, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import AbstractContextManager
from typing import Any, TypeVar
import torch
import zmq
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.distributed import (
get_dcp_group,
get_pcp_group,
@@ -1414,6 +1415,41 @@ class KVCacheStoreRecvingThread(KVTransferThread):
class MooncakeStoreWorker:
"""Worker-side component for MooncakeStoreConnector."""
@staticmethod
def _create_mem_pool(
extra_config: dict[str, Any], model_config: ModelConfig
) -> torch.cuda.MemPool | None:
pool_type = str(extra_config.get("custom_mem_pool") or "").upper()
if not pool_type:
return None
if pool_type not in ("NVLINK", "BAREX"):
raise ValueError(
f"Unsupported custom_mem_pool={pool_type!r}, "
"expected 'NVLINK' or 'BAREX'"
)
if model_config.enable_sleep_mode or model_config.enable_cumem_allocator:
raise ValueError(
"custom_mem_pool is incompatible with enable_sleep_mode "
"or enable_cumem_allocator; CuMemAllocator cannot manage "
"allocations from the custom pool."
)
try:
if pool_type == "NVLINK":
from mooncake.allocator import NVLinkAllocator as allocator_cls
else: # pool_type == "BAREX"
from mooncake.allocator import BarexAllocator as allocator_cls
except ImportError as e:
raise ImportError(
f"custom_mem_pool={pool_type!r} requires "
"mooncake-transfer-engine>=0.3.8. Please upgrade Mooncake."
) from e
device = torch.device("cuda", torch.accelerator.current_device_index())
allocator = allocator_cls.get_allocator(device)
mem_pool = torch.cuda.MemPool(allocator.allocator())
logger.info("Using Mooncake custom memory pool: %s", pool_type)
return mem_pool
def __init__(
self,
vllm_config: VllmConfig,
@@ -1471,6 +1507,10 @@ class MooncakeStoreWorker:
# Initialize MooncakeDistributedStore with its own TransferEngine
store_config = MooncakeStoreConfig.load_from_config()
# Validate and create the custom memory pool before initializing the
# store so failures cannot leak a partially initialized store handle.
self._mem_pool = self._create_mem_pool(extra_config, model_config)
self.store = MooncakeDistributedStore()
local_ip = get_ip()
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
@@ -1741,6 +1781,13 @@ class MooncakeStoreWorker:
)
return token_dbs
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a context manager for the custom MemPool, or None if
no custom pool is configured."""
if self._mem_pool is None:
return None
return torch.cuda.use_mem_pool(self._mem_pool)
def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int:
if self.dcp_size > 1:
return 1
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from collections.abc import Callable, Iterable
from contextlib import AbstractContextManager
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
@@ -294,6 +295,26 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
# ==============================
# Worker-side methods
# ==============================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Forward a custom KV cache memory pool from a child connector.
KV cache is allocated once and can only live in one pool, so at
most one child may provide a context.
"""
found: list[tuple[str, AbstractContextManager]] = []
for connector in self._connectors:
if (ctx := connector.get_mem_pool_context()) is not None:
found.append((type(connector).__name__, ctx))
if len(found) > 1:
names = [name for name, _ in found]
raise ValueError(
f"Multiple connectors provide a KV cache memory pool {names}; "
"KV cache is allocated once and can only live in one pool. "
"Configure custom_mem_pool on at most one connector."
)
return found[0][1] if found else None
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
for c in self._connectors:
c.start_load_kv(forward_context, **kwargs)
@@ -839,6 +839,8 @@ class NixlBaseConnectorWorker:
self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config(
"engine_ttl", 3600.0
)
# Map of pull peer (host, port) -> engine_id currently serving it.
self._engine_by_address: dict[tuple[str, int], EngineId] = {}
self.model_config = vllm_config.model_config
@@ -1300,6 +1302,7 @@ class NixlBaseConnectorWorker:
self._remote_agents[eid] = remote_agents
self._engine_clock_offset[eid] = clock_offset
self._engine_last_active[eid] = time.perf_counter()
self._track_remote_engine_replacement(eid, host, port)
except Exception as e:
self._log_failure(
failure_type="handshake_setup_failed",
@@ -2953,6 +2956,7 @@ class NixlBaseConnectorWorker:
# Handle timeout to avoid stranding blocks on remote.
self._reap_expired_send_leases(done_sending)
self._cleanup_replaced_remote_engines()
return KVConnectorTransferResults(
finished_sending=done_sending,
finished_recving=done_recving,
@@ -3541,14 +3545,53 @@ class NixlBaseConnectorWorker:
and meta.remote is not None
}
def _track_remote_engine_replacement(
self, engine_id: EngineId, host: str, port: int
) -> None:
"""Record the engine currently serving a pull peer address."""
# TODO: Also handle push mode, which handshakes in both directions.
if self._TRANSFER_MODE != "pull":
return
previous = self._engine_by_address.get((host, port))
self._engine_by_address[(host, port)] = engine_id
if previous is None or previous == engine_id:
return
logger.info(
"Remote engine at %s:%d changed from %s to %s.",
host,
port,
previous,
engine_id,
)
def _cleanup_replaced_remote_engines(self) -> None:
"""Release replaced pull peers once requests and handshakes have drained."""
if self._TRANSFER_MODE != "pull":
return
with self._handshake_lock:
if self._handshake_futures:
return
replaced = self._remote_agents.keys() - self._engine_by_address.values()
if not replaced:
return
# _recving_metadata is only accessed from this (main) thread.
busy = {
meta.remote.engine_id
for meta in self._recving_metadata.values()
if meta.remote is not None
}
for engine_id in replaced - busy:
self._cleanup_remote_engine(engine_id, log_eviction=False)
logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
def _cleanup_remote_engine(
self, engine_id: EngineId, *, log_eviction: bool = True
) -> None:
"""Remove all state for a single remote engine.
Releases NIXL resources (dlist handles, remote agents) and clears
all per-engine data structures. Used by both TTL eviction and
shutdown.
all per-engine data structures. Used by TTL eviction, replaced peer
cleanup and shutdown.
"""
assert engine_id in self._remote_agents
@@ -3558,6 +3601,9 @@ class NixlBaseConnectorWorker:
# Pop under the handshake lock; NIXL teardown stays outside it.
with self._handshake_lock:
agents = self._remote_agents.pop(engine_id)
for address, eid in list(self._engine_by_address.items()):
if eid == engine_id:
del self._engine_by_address[address]
for agent_name in agents.values():
self.nixl_wrapper.remove_remote_agent(agent_name)
@@ -84,7 +84,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
# Start transfers for requests whose handshakes have now finished.
while not self._ready_requests.empty():
self._read_blocks_for_req(*self._ready_requests.get_nowait())
req_id, meta = self._ready_requests.get_nowait()
assert meta.remote is not None
if meta.remote.engine_id not in self._remote_agents:
# The engine was released after its handshake completed, so
# handshake again. This fails if the engine is gone.
self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
continue
self._read_blocks_for_req(req_id, meta)
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
@@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
@@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import (
from vllm.v1.simple_kv_offload.metadata import (
SimpleCPUOffloadMetadata,
)
from vllm.v1.simple_kv_offload.metrics import (
SimpleCPUOffloadPromMetrics,
SimpleCPUOffloadStats,
)
from vllm.v1.simple_kv_offload.worker import (
SimpleCPUOffloadWorker,
)
@@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
hash_block_size=hash_block_size,
lazy_offload=lazy_offload,
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
use_page_cache=use_page_cache if disk_mode else False,
)
elif role == KVConnectorRole.WORKER:
self.worker_handler = SimpleCPUOffloadWorker(
@@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
return self.scheduler_manager.get_boundary_store_stats()
return None
def get_kv_connector_stats(self) -> KVConnectorStats | None:
# Worker-side callers may invoke this hook on connectors constructed
# via __new__ (as in test_worker.py), which lack scheduler_manager.
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
self, "scheduler_manager", None
)
if scheduler_manager is not None:
return scheduler_manager.get_stats()
return None
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
SimpleCPUOffloadStats(data=data)
if data is not None
else SimpleCPUOffloadStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return SimpleCPUOffloadPromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def take_events(self) -> Iterable[KVCacheEvent]:
if self.scheduler_manager is not None:
return self.scheduler_manager.take_events()
+8
View File
@@ -1583,6 +1583,14 @@ def get_dcp_group() -> GroupCoordinator:
return _DCP
def get_dcp_world_size_and_rank(enabled: bool = True) -> tuple[int, int]:
"""Return ``(world_size, rank)`` in the DCP group, or ``(1, 0)`` when disabled
(e.g. a replicated draft cache) or the group is uninitialized (unit tests)."""
if not enabled or _DCP is None:
return 1, 0
return _DCP.world_size, _DCP.rank_in_group
_PP: GroupCoordinator | None = None
+8 -13
View File
@@ -141,7 +141,7 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
VLLM_ROCM_USE_AITER_MOE: bool = True
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False
VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto"
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
VLLM_ROCM_USE_AITER_MLA: bool = True
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
@@ -1280,18 +1280,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_USE_AITER_MOE": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
),
# Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4
# activations, separated gate/up layout) instead of default a16w4. vLLM
# sets AITER_SITUV2_A4W4 at init when this flag is on and clears any
# legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first).
# VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing
# recipes; it does not select a8w4 kernels.
# Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag
# and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics.
"VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1")
or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower()
in ("true", "1")
# Activation dtype for the Kimi-K3 SiTU MXFP4 MoE (AITER FlyDSL SiTUv2):
# auto (= a4w4), a4w4, a8w4 or a16w4. Legacy 1/0 mean a4w4/a16w4.
"VLLM_ROCM_USE_AITER_MOE_SITUV2": env_with_choices(
"VLLM_ROCM_USE_AITER_MOE_SITUV2",
"auto",
["auto", "a4w4", "a8w4", "a16w4", "0", "1"],
case_sensitive=False,
),
# MoE sorting dispatch policy for AITER fused MoE kernels.
# 0 = auto (default): single-pass for small batches, multi-pass
@@ -385,10 +385,11 @@ def rocm_aiter_fused_experts(
gate_mode = ""
if activation == MoEActivation.SITU:
# SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4
# fp4 activations with separated gate/up weights (AITER #4463);
# default a16w4 SiTU also stays separated.
gate_mode = GateMode.SEPARATED.value
gate_mode = (
GateMode.INTERLEAVE.value
if rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
else GateMode.SEPARATED.value
)
elif quant_config.use_mxfp4_w4a16:
gate_mode = GateMode.INTERLEAVE.value
elif activation_interleave is not None:
@@ -1596,8 +1596,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
# SiTUv2 flydsl uses separated gate/up layout (a4w4).
guinterleave = False
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
w13_weight.data.view(fp4_dtype), 16, guinterleave
)
+2 -2
View File
@@ -697,7 +697,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
else:
gemm_rs.warn_incompatible_projection()
def _wo_b_proj(self, z: torch.Tensor) -> torch.Tensor:
def _wo_b_proj(self, z: torch.Tensor | QuantizedActivation) -> torch.Tensor:
"""Apply ``wo_b``; with GEMM-RS bound, also reduce-scatter the result.
Every ``_o_proj`` implementation projects through this so the decoder
@@ -707,7 +707,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
"""
if self.gemm_rs is None:
return self.wo_b(z)
if self.gemm_rs.should_run(z):
if isinstance(z, torch.Tensor) and self.gemm_rs.should_run(z):
return self.gemm_rs.apply(z, self.wo_b)
# Small batches stay on the unfused path, which is faster there.
return sp_reduce_scatter(self.wo_b(z))
@@ -9,15 +9,16 @@ import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
compute_fp8_einsum_recipe,
deep_gemm_fp8_o_proj,
)
from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.common.ops import (
build_flashinfer_mixed_sparse_indices,
compute_global_topk_indices_and_lens,
)
from vllm.models.deepseek_v41.nvidia.ops.o_proj import (
dsv41_o_proj,
register_dsv41_o_proj_warmup,
)
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4FlashMLAMetadata,
DeepseekV4SparseMLABackend,
@@ -242,27 +243,14 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
return _pad_to_supported_q_heads(num_heads)
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
self.rotary_emb.cos_sin_cache,
self.wo_a,
self._wo_b_proj,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
o_lora_rank=self.o_lora_rank,
einsum_recipe=self._einsum_recipe,
tma_aligned_scales=self._tma_aligned_scales,
)
return dsv41_o_proj(self, attn_out, positions)
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe(
self._o_proj_block_size
)
register_dsv41_o_proj_warmup(self)
# Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the
# per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``.
if self.kv_cache_torch_dtype != torch.float8_e4m3fn:
@@ -623,21 +611,7 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
return _pad_to_supported_q_heads(num_heads)
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
self.rotary_emb.cos_sin_cache,
self.wo_a,
self._wo_b_proj,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
o_lora_rank=self.o_lora_rank,
einsum_recipe=self._einsum_recipe,
tma_aligned_scales=self._tma_aligned_scales,
)
return dsv41_o_proj(self, attn_out, positions)
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
super().__init__(vllm_config, *args, **kwargs)
+7 -19
View File
@@ -6,16 +6,17 @@ from typing import TYPE_CHECKING, ClassVar, cast
import torch
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
compute_fp8_einsum_recipe,
deep_gemm_fp8_o_proj,
)
from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.common.ops import (
combine_topk_swa_indices,
compute_global_topk_indices_and_lens,
dequantize_and_gather_k_cache,
)
from vllm.models.deepseek_v41.nvidia.ops.o_proj import (
dsv41_o_proj,
register_dsv41_o_proj_warmup,
)
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4FlashMLABackend,
DeepseekV4FlashMLAMetadata,
@@ -57,23 +58,10 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe(
self._o_proj_block_size
)
register_dsv41_o_proj_warmup(self)
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
self.rotary_emb.cos_sin_cache,
self.wo_a,
self._wo_b_proj,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
o_lora_rank=self.o_lora_rank,
einsum_recipe=self._einsum_recipe,
tma_aligned_scales=self._tma_aligned_scales,
)
return dsv41_o_proj(self, attn_out, positions)
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
@@ -0,0 +1,599 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DSV4.1 small-batch MXFP8 WO-A with inverse RoPE and quantization."""
from dataclasses import dataclass
from typing import Any
import cutlass
import cutlass.cute as cute
import cutlass.utils as utils
import torch
from cuda.bindings.driver import CUstream
from cutlass import (
BFloat16,
Float8E4M3FN,
Float32,
Int32,
Int64,
Uint8,
Uint32,
Uint64,
)
from cutlass._mlir.dialects import llvm
from cutlass.cute.nvgpu import cpasync, tcgen05
from cutlass.cute.runtime import make_fake_tensor
from cutlass.cutlass_dsl import dsl_user_op
from vllm.cute_utils import _tcgen05, mbarrier, recast_val, simple_tma_copy
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, kernel_launcher
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
)
# Each head is [nope | rope] as in fused_inv_rope_fp8_quant: only the last
# _ROPE_DIM columns carry RoPE.
_HEAD_DIM = 512
_ROPE_DIM = 64
@dsl_user_op
def _ue8m0_rp(value, *, loc=None, ip=None):
result = llvm.inline_asm(
Uint32.mlir_type,
[value.ir_value(loc=loc, ip=ip)],
"{ .reg .b16 sf; cvt.rp.satfinite.ue8m0x2.f32 sf, $1, $1; "
"cvt.u32.u16 $0, sf; and.b32 $0, $0, 255; }",
"=r,f",
has_side_effects=False,
loc=loc,
ip=ip,
)
return Uint32(result)
@dsl_user_op
def _rope_fma(value, cos, other, sin, *, subtract=False, loc=None, ip=None):
"""Preserve multiply/FMA rounding when RoPE loads are hoisted."""
result = llvm.inline_asm(
Float32.mlir_type,
[v.ir_value(loc=loc, ip=ip) for v in (value, cos, other, sin)],
"{ .reg .f32 product; mul.rn.f32 product, $3, $4; "
+ (
"neg.f32 product, product; fma.rn.f32 $0, $1, $2, product; }"
if subtract
else "fma.rn.f32 $0, $1, $2, product; }"
),
"=f,f,f,f,f",
has_side_effects=False,
loc=loc,
ip=ip,
)
return Float32(result)
@cute.jit
def _scale(amax: Float32):
exponent = _ue8m0_rp(amax * Float32(1.0 / 448.0))
inv = recast_val((Uint32(254) - exponent) << 23, Float32)
if exponent == 0:
inv = Float32(0)
return exponent, inv
class FusedWoAKernel(VllmCuTeDSLJitKernel["FusedWoAKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
# T=1..96 uses at most 46 keys per (n_groups, heads_per_group, o_lora_rank).
tokens_per_tile: int
token_tiles: int
full_tiles: bool
n_groups: int
heads_per_group: int
o_lora_rank: int
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
tokens = compile_key.tokens_per_tile
token_tiles = compile_key.token_tiles
full_tiles = compile_key.full_tiles
acc_cols = 1 << (tokens - 1).bit_length()
tile_m = max(8, acc_cols)
tile_n = 128
tile_k = 128
# tcgen05 MXFP8 MMA consumes K = 32, one MX block, per instruction.
mma_k = 32
# Each group's heads split K across one cluster, one head per CTA.
heads_per_group = compile_key.heads_per_group
tiles_per_group = compile_key.o_lora_rank // tile_n
n_tiles = compile_key.n_groups * tiles_per_group
# Each CTA keeps one whole head of K in SMEM, one TMA box per stage.
num_stages = _HEAD_DIM // tile_k
w_stage_bytes = tile_n * tile_k
nope_dim = _HEAD_DIM - _ROPE_DIM
threads_per_token = _HEAD_DIM // 4
sf_col = max(16, tile_m)
tmem_cols = sf_col * 2
threads = min(acc_cols * threads_per_token, 512)
tokens_per_iter = threads // threads_per_token
@cute.kernel
def device_kernel(
x: cute.Tensor,
positions: cute.Tensor,
rope: cute.Tensor,
w: cpasync.TmaInfo,
ws: cpasync.TmaInfo,
q: cute.Tensor,
qs: cute.Tensor,
num_tokens: Int32,
):
total_tokens = num_tokens
if cutlass.const_expr(full_tiles):
total_tokens = Int32(tokens * token_tiles)
tid, _, _ = cute.arch.thread_idx()
bid, token_tile, _ = cute.arch.block_idx()
token_start = Int32(0)
if cutlass.const_expr(token_tiles > 1):
token_start = token_tile * tokens
warp = cute.arch.make_warp_uniform(tid // 32)
lane = tid % 32
split = bid % heads_per_group
tile = bid // heads_per_group
group = tile // tiles_per_group
smem = utils.SmemAllocator()
sW = smem.allocate_tensor(
Float8E4M3FN,
w.smem_layout.outer,
byte_alignment=128,
swizzle=w.smem_layout.inner,
)
sX = smem.allocate_tensor(
Float8E4M3FN,
cute.make_layout(
(tile_m, tile_k, num_stages), stride=(tile_k, 1, tile_m * tile_k)
),
byte_alignment=1024,
swizzle=cute.make_swizzle(3, 4, 3),
)
# tcgen05.cp 32x128b.warpx4 order: row r at byte (r % 32) * 16 +
# (r // 32) * 4, one 4-byte cell of MX-block scales per row.
sW_SF = smem.allocate_tensor(
Int32,
cute.make_layout(((32, 4), num_stages), stride=((4, 1), tile_n)),
128,
)
sW_SF_raw = smem.allocate_tensor(
Int32, cute.make_layout((tile_n, num_stages)), 128
)
sX_SF = smem.allocate_tensor(
Uint8,
cute.make_layout(
((32, 4), tile_k // mma_k, num_stages),
stride=((16, 4), 1, tile_n * 4),
),
128,
)
copy_fp8x4 = cute.make_copy_atom(
cute.nvgpu.CopyUniversalOp(), Float8E4M3FN, num_bits_per_copy=32
)
partial = smem.allocate_tensor(
Float32,
cute.make_layout(
(tile_n, cute.ceil_div(tokens, heads_per_group), heads_per_group)
),
128,
)
mbar_loaded = smem.allocate_array(Int64, num_stages)
mbar_ws_loaded = smem.allocate_array(Int64, 1)
mbar_done = smem.allocate_array(Int64, 1)
mbar_reduced = smem.allocate_array(Int64, 1)
taddr = smem.allocate(Int32, 4)
if tid == 0:
for stage in cutlass.range_constexpr(num_stages):
# Wait for weight TMA and this stage's quantization threads.
cute.arch.mbarrier_init(mbar_loaded + stage, 1 + threads // 4)
cute.arch.mbarrier_init(mbar_ws_loaded, 1)
cute.arch.mbarrier_init(mbar_done, 1)
cute.arch.mbarrier_init(mbar_reduced, 1)
cute.arch.mbarrier_init_fence()
# Local users of the barriers (warp 0's TMA below) need the init too.
cute.arch.sync_threads()
# Paired with cluster_wait() below: peers must see mbar_reduced
# initialized before their st.async; the split hides the barrier.
cute.arch.cluster_arrive_relaxed()
if warp == 0:
# Raw DeepGEMM MN-major scales; the MMA warp transposes them to
# the UTCCP order, which TMA cannot scatter at 4-byte granularity.
ws_tiles = cute.zipped_divide(ws.tma_tensor, (tile_n, num_stages))
with cute.arch.elect_one():
mbarrier.arrive_expect_tx(mbar_ws_loaded, tile_n * num_stages * 4)
simple_tma_copy(
ws.atom,
ws_tiles[None, (tile % tiles_per_group, split, group)],
sW_SF_raw,
mbar_ws_loaded,
)
tiles = cute.zipped_divide(w.tma_tensor, (tile_n, tile_k))
for stage in cutlass.range_constexpr(num_stages):
with cute.arch.elect_one():
mbarrier.arrive_expect_tx(mbar_loaded + stage, w_stage_bytes)
simple_tma_copy(
w.atom,
tiles[None, (tile, split * num_stages + stage)],
sW[None, None, stage],
mbar_loaded + stage,
)
# x and positions come from the PDL predecessor; everything above
# only touches weights. Padded sX rows only feed unread MMA columns.
cute.arch.griddepcontrol_wait()
# TMEM is allocated at runtime, and a PDL predecessor on this SM may
# hold it until it exits; allocating earlier can deadlock.
if warp == 0:
cute.arch.alloc_tmem(tmem_cols, taddr)
cute.arch.relinquish_tmem_alloc_permit()
copy_x = cute.make_copy_atom(
cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=64
)
n_iters = cute.ceil_div(tokens, tokens_per_iter)
qid = (tid + 32) % threads_per_token
k = qid * 4
# Tail rows repeat the last input; their output stores are masked.
# Issue every token's x load before any math so their latencies overlap.
x_bf16 = cute.make_rmem_tensor((4, n_iters), BFloat16)
for iteration in cutlass.range_constexpr(n_iters):
token = iteration * tokens_per_iter + tid // threads_per_token
if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens:
x_src = cute.local_tile(
x[
cute.min(token_start + token, total_tokens - 1),
group * heads_per_group + split,
None,
],
(4,),
(qid,),
)
cute.copy(copy_x, x_src, x_bf16[None, iteration])
# Overlap the RoPE loads across tokens before consuming any of them.
cos_reg = cute.make_rmem_tensor((2, n_iters), Float32)
sin_reg = cute.make_rmem_tensor((2, n_iters), Float32)
if cutlass.const_expr(n_iters > 1):
copy_rope = cute.make_copy_atom(
cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=32
)
if k >= nope_dim:
for iteration in cutlass.range_constexpr(n_iters):
token = iteration * tokens_per_iter + tid // threads_per_token
if (
cutlass.const_expr(tokens % tokens_per_iter == 0)
or token < tokens
):
src_token = cute.min(token_start + token, total_tokens - 1)
pos = positions[src_token]
freq = (k - nope_dim) // 2
rope_row = rope[pos, None]
cos_src = cute.local_tile(rope_row, (2,), (freq // 2,))
cute.copy(copy_rope, cos_src, cos_reg[None, iteration])
sin_tile = (freq + _ROPE_DIM // 2) // 2
sin_src = cute.local_tile(rope_row, (2,), (sin_tile,))
cute.copy(copy_rope, sin_src, sin_reg[None, iteration])
for iteration in cutlass.range_constexpr(n_iters):
token = iteration * tokens_per_iter + tid // threads_per_token
if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens:
x_f32 = cute.make_rmem_tensor((4,), Float32)
x_f32.store(x_bf16[None, iteration].load().to(Float32))
if k >= nope_dim:
# Inverse RoPE on the interleaved pairs (k, k+1) and
# (k+2, k+3); each rope row is cos || sin.
for pair in cutlass.range_constexpr(2):
if cutlass.const_expr(n_iters > 1):
cos = cos_reg[pair, iteration]
sin = sin_reg[pair, iteration]
else:
pos = positions[
cute.min(token_start + token, total_tokens - 1)
]
freq = (k - nope_dim) // 2 + pair
cos = Float32(rope[pos, freq])
sin = Float32(rope[pos, freq + _ROPE_DIM // 2])
even, odd = x_f32[2 * pair], x_f32[2 * pair + 1]
x_f32[2 * pair] = _rope_fma(even, cos, odd, sin)
x_f32[2 * pair + 1] = _rope_fma(
odd, cos, even, sin, subtract=True
)
amax = cute.arch.fmax(
cute.arch.fmax(cute.abs(x_f32[0]), cute.abs(x_f32[1])),
cute.arch.fmax(cute.abs(x_f32[2]), cute.abs(x_f32[3])),
)
amax = cute.arch.warp_reduction_max(amax, threads_in_group=8)
exponent, inv = _scale(cute.arch.fmax(amax, Float32(1e-10)))
x_fp8 = cute.make_rmem_tensor((4,), Float8E4M3FN)
x_fp8.store((x_f32.load() * inv).to(Float8E4M3FN))
cute.copy(
copy_fp8x4,
x_fp8,
cute.local_tile(
sX[token, None, k // tile_k], (4,), (qid % 32,)
),
)
if lane % 8 == 0:
sX_SF[token, k % tile_k // mma_k, k // tile_k] = Uint8(exponent)
cute.arch.fence_proxy("async.shared", space="cta")
mbarrier.arrive(mbar_loaded + qid // 32, order="release")
if warp == 0:
base = cute.make_tensor(taddr, cute.make_layout(1))[0]
# sW / sX are 128B-swizzled K-major; one stage's K fills the
# swizzle atom, so the leading byte offset is unused.
sdesc = _tcgen05.make_sdesc_128B_swizzle(LBO=0)
# Unswizzled 32 x 16 B scale tiles: SBO = one 8-row core matrix
# (16 B units at bit 32); bit 46 is the SM100 descriptor version.
sf_sbo = 8 * 16
sfdesc = Uint64((sf_sbo >> 4 << 32) | (1 << 46))
# M = tile_n weight rows (A), N = tile_m token rows (B).
idesc = _tcgen05.make_mxfp8_idesc(tile_n, tile_m)
# sW_SF's layout is the UTCCP order, so a plain copy transposes;
# fence the generic-proxy writes before tcgen05.cp reads them.
cute.arch.mbarrier_wait(mbar_ws_loaded, 0)
for stage in cutlass.range_constexpr(num_stages):
for i in cutlass.range_constexpr(4):
sW_SF[i * 32 + lane, stage] = sW_SF_raw[i * 32 + lane, stage]
cute.arch.fence_proxy("async.shared", space="cta")
cute.arch.sync_warp()
for stage in cutlass.range_constexpr(num_stages):
cute.arch.mbarrier_wait(mbar_loaded + stage, 0)
_tcgen05.fence_after_thread_sync()
if cutlass.const_expr(stage == num_stages - 1):
with cute.arch.elect_one():
cute.arch.griddepcontrol_launch_dependents()
adesc = sdesc | (sW[None, None, stage].iterator.toint() >> 4)
bdesc = sdesc | (sX[None, None, stage].iterator.toint() >> 4)
_tcgen05.cp(
base + sf_col,
sfdesc | (sW_SF[None, stage].iterator.toint() >> 4),
"32x128b",
"warpx4",
)
_tcgen05.cp(
base + sf_col + 4,
sfdesc | (sX_SF[None, None, stage].iterator.toint() >> 4),
"32x128b",
"warpx4",
)
# Advance mma_k FP8 bytes (16 B units) and pick MX block kk of
# each 4-byte scale (B sf_id at bit 4, A sf_id at bit 29).
for kk in cutlass.range_constexpr(tile_k // mma_k):
_tcgen05.mma_mxfp8(
base,
adesc + kk * mma_k // 16,
bdesc + kk * mma_k // 16,
idesc + ((kk << 4) | (kk << 29)),
base + sf_col,
base + sf_col + 4,
stage > 0 or kk > 0,
)
_tcgen05.commit(mbar_done)
# Only the MMA warp waits; barrier 2 releases the TMEM readers.
cute.arch.mbarrier_wait(mbar_done, 0)
if tid < tile_n:
cute.arch.barrier(barrier_id=2, number_of_threads=tile_n)
# Tokens t with t % heads_per_group == split arrive from every peer
# (this CTA included), tile_n Float32 rows each.
if tid == 0:
owned = cute.ceil_div(tokens - split, heads_per_group)
mbarrier.arrive_expect_tx(
mbar_reduced, owned * tile_n * heads_per_group * 4
)
# Pairs with cluster_arrive_relaxed() above: every peer's
# mbar_reduced is initialized before the st.async below.
cute.arch.cluster_wait()
# quack's process-wide const_expr-if rewrite can leave these unbound
# after the dynamic path above; bind them so the regions join.
amax, exponent, inv = Float32(0), Uint32(0), Float32(0)
if tid < tile_n:
base = cute.make_tensor(taddr, cute.make_layout(1))[0]
_tcgen05.fence_after_thread_sync()
acc = cute.make_rmem_tensor(acc_cols, Float32)
if cutlass.const_expr(tokens == 1):
acc[0] = _tcgen05.ld(warp * 32, base, "32x32b", 1)
else:
acc.store(_tcgen05.ld(warp * 32, base, "32x32b", acc_cols))
_tcgen05.wait_ld()
for token in cutlass.range_constexpr(tokens):
ptr = cute.domain_offset(
(tid, token // heads_per_group, split), partial
).iterator
cute.arch.store_async_dsmem(
ptr,
recast_val(acc[token], Int32),
mbar_reduced,
token % heads_per_group,
)
if split < tokens:
# One warp waits for the peers' bytes; named barrier 1 (0 is
# sync_threads) releases the other tile_n epilogue threads.
if warp == 0:
cute.arch.mbarrier_wait(mbar_reduced, 0)
cute.arch.barrier(barrier_id=1, number_of_threads=tile_n)
for i in cutlass.range_constexpr(
cute.ceil_div(tokens, heads_per_group)
):
token = split + i * heads_per_group
if cutlass.const_expr(
full_tiles
and (
tokens <= heads_per_group
or tokens % heads_per_group == 0
)
) or (token < tokens and token_start + token < total_tokens):
value = Float32(0)
for peer in cutlass.range_constexpr(heads_per_group):
value += partial[tid, i, peer]
value = Float32(BFloat16(value))
amax = cute.arch.warp_redux_sync(value, "max", abs=True)
exponent, inv = _scale(amax)
q[token_start + token, tile * tile_n + tid] = Float8E4M3FN(
value * inv
)
if lane == 0:
row = token_start + token
offset = row % 32 * 16 + row // 32 * 4 + warp
qs[tile * 512 + offset] = Uint8(exponent)
# Every scale tile owns its padding, with no second memset kernel.
for i in cutlass.range_constexpr(4):
offset = i * tile_n + tid
row = offset // 16 + (offset % 16 // 4) * 32
if token_tile == 0 and split == 0 and row >= total_tokens:
qs[tile * 512 + offset] = Uint8(0)
# Each receiver drained all writes into its own inbox. Only local
# TMEM readers must finish before deallocation; no peer reads SMEM.
if tid < tile_n:
cute.arch.barrier(barrier_id=2, number_of_threads=tile_n)
if warp == 0:
cute.arch.dealloc_tmem(
cute.make_ptr(
Float32,
cute.make_tensor(taddr, cute.make_layout(1))[0],
cute.AddressSpace.tmem,
),
tmem_cols,
)
@cute.jit
def host_entrypoint(
x: cute.Tensor,
positions: cute.Tensor,
rope: cute.Tensor,
w: cute.Tensor,
ws: cute.Tensor,
q: cute.Tensor,
qs: cute.Tensor,
num_tokens: Int32,
stream: CUstream,
):
layout = cute.make_composed_layout(
cute.make_swizzle(3, 4, 3),
0,
cute.make_layout(
(tile_n, tile_k, num_stages), stride=(tile_k, 1, w_stage_bytes)
),
)
tma = cpasync.make_tiled_tma_atom(
cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE),
w,
layout,
(tile_n, tile_k),
)
ws_tma = cpasync.make_tiled_tma_atom(
cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE),
cute.make_tensor(ws.iterator, cute.select(ws.layout, mode=[1, 2, 0])),
cute.make_layout((tile_n, num_stages)),
(tile_n, num_stages),
)
device_kernel(x, positions, rope, tma, ws_tma, q, qs, num_tokens).launch(
grid=(n_tiles * heads_per_group, token_tiles, 1),
block=(threads, 1, 1),
cluster=(heads_per_group, 1, 1),
stream=stream,
use_pdl=True,
)
return host_entrypoint
def dispatch( # type: ignore[override]
self, *, tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int
) -> CompileKey:
token_tiles = (tokens + 31) // 32
# Host integer arithmetic: balance tiles in multiples of four tokens.
tile_tokens = (
tokens
if token_tiles == 1
else (tokens + 4 * token_tiles - 1) // (4 * token_tiles) * 4
)
return self.CompileKey(
tokens_per_tile=tile_tokens,
token_tiles=token_tiles,
full_tiles=tokens % tile_tokens == 0,
n_groups=n_groups,
heads_per_group=heads_per_group,
o_lora_rank=o_lora_rank,
)
def get_warmup_keys(
self, *, max_tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int
) -> list[CompileKey]:
# Trace all accepted token counts and deduplicate shared tile shapes.
return self._trace_dispatch(self.dispatch)(
tokens=WarmupIntRange(1, max_tokens + 1),
n_groups=n_groups,
heads_per_group=heads_per_group,
o_lora_rank=o_lora_rank,
)
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
tokens = (
compile_key.tokens_per_tile * compile_key.token_tiles
if compile_key.full_tiles
else cute.sym_int()
)
groups = compile_key.n_groups
rank = compile_key.o_lora_rank
n = groups * rank
k = compile_key.heads_per_group * _HEAD_DIM
return (
make_fake_tensor(
BFloat16,
(tokens, groups * compile_key.heads_per_group, _HEAD_DIM),
(cute.sym_int(divisibility=8), _HEAD_DIM, 1),
assumed_align=16,
),
make_fake_tensor(Int64, (tokens,), (1,)),
make_fake_tensor(Float32, (cute.sym_int(), _ROPE_DIM), (_ROPE_DIM, 1)),
make_fake_tensor(Float8E4M3FN, (n, k), (k, 1), assumed_align=16),
make_fake_tensor(
Int32,
(groups, rank, k // 128),
(rank * k // 128, 1, rank),
assumed_align=16,
),
make_fake_tensor(Float8E4M3FN, (tokens, n), (n, 1)),
make_fake_tensor(Uint8, (128 * n // 32,), (1,)),
Int32(0),
)
@kernel_launcher
def __call__(
self,
*,
x: torch.Tensor,
positions: torch.Tensor,
rope: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
) -> CuTeDSLLaunchSpec["FusedWoAKernel.CompileKey"]:
"""Return FP8 WO-A output and FlashInfer F8_128x4 scale bytes."""
tokens = x.shape[0]
groups, rank = weight_scale.shape[:2]
n = groups * rank
q = torch.empty((tokens, n), device=x.device, dtype=torch.float8_e4m3fn)
scales = torch.empty(128 * n // 32, device=x.device, dtype=torch.uint8)
launch_args = (
x,
positions,
rope,
weight.view(n, -1),
weight_scale,
q,
scales,
tokens,
)
compile_key = self.dispatch(
tokens=tokens,
n_groups=groups,
heads_per_group=x.shape[1] // groups,
o_lora_rank=rank,
)
return compile_key, launch_args, (q, scales)
_FUSED_WO_A_KERNEL = FusedWoAKernel()
@@ -0,0 +1,95 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DSV4.1 output projection with a small-batch SM100/SM103 fusion."""
import torch
from torch import nn
from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
FlashInferCutedslMxfp8LinearKernel,
)
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj
from vllm.platforms import current_platform
def _fused_wo_a_max_tokens(layer: nn.Module) -> int:
max_tokens = getattr(layer, "_fused_wo_a_token_limit", None)
if max_tokens is None:
# Additional token tiles only pay off for smaller grids.
ctas_per_tile = layer.n_local_heads * (layer.o_lora_rank // 128)
max_tokens = 96 if ctas_per_tile <= 128 else 32
layer._fused_wo_a_token_limit = max_tokens
return max_tokens
def _can_fuse_wo_a(layer: nn.Module) -> bool:
"""Whether the attention layer matches the fused WO-A kernel's layout."""
method = getattr(layer.wo_b, "scheme", layer.wo_b.quant_method)
return (
current_platform.is_device_capability_family(100)
# WO-A is FP8 with per-32 scales, and WO-B takes MXFP8 input directly.
and layer._einsum_recipe == (1, 1, 32)
and get_input_quant_key(layer.wo_b) == kMxfp8Dynamic
and isinstance(
getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel
)
# A group's heads form one portable (<= 8 CTA) cluster.
and layer.n_local_heads // layer.n_local_groups <= 8
and (layer.nope_head_dim, layer.rope_head_dim) == (448, 64)
and layer.o_lora_rank % 128 == 0
)
def register_dsv41_o_proj_warmup(layer: nn.Module) -> None:
"""Warm the fused WO-A token counts ``dsv41_o_proj`` can dispatch."""
# Draft models load outside the warmup registry; their layers share the
# target's attention shapes, so the target's registration covers them too.
if _can_fuse_wo_a(layer):
from .fused_wo_a import _FUSED_WO_A_KERNEL
_FUSED_WO_A_KERNEL.register_warmup(
max_tokens=_fused_wo_a_max_tokens(layer),
n_groups=layer.n_local_groups,
heads_per_group=layer.n_local_heads // layer.n_local_groups,
o_lora_rank=layer.o_lora_rank,
)
def dsv41_o_proj(
layer: nn.Module, attn_out: torch.Tensor, positions: torch.Tensor
) -> torch.Tensor:
"""``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100/SM103 batches."""
o = attn_out[:, : layer.n_local_heads, :]
cos_sin_cache = layer.rotary_emb.cos_sin_cache
if 1 <= o.shape[0] <= _fused_wo_a_max_tokens(layer) and _can_fuse_wo_a(layer):
from .fused_wo_a import _FUSED_WO_A_KERNEL
q, scales = _FUSED_WO_A_KERNEL(
x=o,
positions=positions,
rope=cos_sin_cache,
weight=layer.wo_a.weight,
weight_scale=layer.wo_a.weight_scale,
)
return layer._wo_b_proj(
QuantizedActivation(q, scales, o.dtype, q.shape, kMxfp8Dynamic)
)
return deep_gemm_fp8_o_proj(
o,
positions,
cos_sin_cache,
layer.wo_a,
layer._wo_b_proj,
n_groups=layer.n_local_groups,
heads_per_group=layer.n_local_heads // layer.n_local_groups,
nope_dim=layer.nope_head_dim,
rope_dim=layer.rope_head_dim,
o_lora_rank=layer.o_lora_rank,
einsum_recipe=layer._einsum_recipe,
tma_aligned_scales=layer._tma_aligned_scales,
)
@@ -383,6 +383,7 @@ def _prefill_logits(
page_table: torch.Tensor,
query_start_loc: torch.Tensor,
visible_blocks: torch.Tensor,
logits_workspace: torch.Tensor,
max_query_len: int,
logits_width: int,
query_offset: int,
@@ -392,10 +393,14 @@ def _prefill_logits(
assert visible_blocks.shape == (q.shape[0],)
assert 0 <= query_offset <= query_offset + num_queries <= q.shape[0]
assert 0 < logits_width <= page_table.shape[1] * k_cache.shape[1]
assert logits_workspace.is_contiguous()
assert logits_workspace.numel() >= num_queries * logits_width
logits = torch.empty(
(num_queries, logits_width), dtype=torch.float32, device=q.device
# slice from pre-allocated workspace
logits = logits_workspace[: num_queries * logits_width].view(
num_queries, logits_width
)
# tuned on GB300
if k_cache.dtype == torch.float8_e4m3fn:
TILE_R, STAGES, num_warps = 32, 2, 8
@@ -610,8 +615,13 @@ def qsa_select_paged_prefill(
logits_width = min(max(64, logits_width), page_table.shape[1] * k_cache.shape[1])
# chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB
# always allocate the worst case to avoid memory fragmentation.
max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
rows_per_chunk = max(1, max_logits_bytes // (logits_width * 4))
budget_bytes = max(max_logits_bytes, page_table.shape[1] * k_cache.shape[1] * 4)
logits_workspace = q.new_empty(budget_bytes // 4, dtype=torch.float32)
topk_workspace = torch.empty(
(_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device
)
@@ -625,6 +635,7 @@ def qsa_select_paged_prefill(
page_table,
query_start_loc,
visible_blocks,
logits_workspace,
max_query_len,
logits_width,
query_offset=query_start,
+7 -8
View File
@@ -870,15 +870,14 @@ class AttentionImplBase(ABC, Generic[T]):
def __new__(cls, *args, **kwargs):
# use __new__ so that all subclasses will call this
self = super().__new__(cls)
try:
from vllm.distributed.parallel_state import get_dcp_group
from vllm.config import get_current_vllm_config_or_none
from vllm.distributed.parallel_state import get_dcp_world_size_and_rank
self.dcp_world_size = get_dcp_group().world_size
self.dcp_rank = get_dcp_group().rank_in_group
except AssertionError:
# DCP might not be initialized in testing
self.dcp_world_size = 1
self.dcp_rank = 0
# Replicated drafts run at DCP=1 inside a DCP target's process group.
config = get_current_vllm_config_or_none()
self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank(
config is None or config.parallel_config.decode_context_parallel_size > 1
)
try:
from vllm.distributed.parallel_state import get_pcp_group
+7 -10
View File
@@ -60,7 +60,10 @@ from vllm.config import (
get_layers_from_vllm_config,
)
from vllm.config.cache import CacheDType
from vllm.distributed.parallel_state import get_dcp_group
from vllm.distributed.parallel_state import (
get_dcp_group,
get_dcp_world_size_and_rank,
)
from vllm.logger import init_logger
from vllm.platforms.interface import DeviceCapability
from vllm.utils.math_utils import cdiv, round_up
@@ -726,15 +729,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
num_qo_heads=self.num_heads_q,
)
try:
from vllm.distributed.parallel_state import get_dcp_group
self.dcp_world_size = get_dcp_group().world_size
self.dcp_rank = get_dcp_group().rank_in_group
except AssertionError:
# DCP might not be initialized in testing
self.dcp_world_size = 1
self.dcp_rank = 0
self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank(
kv_cache_spec.dcp_sharded
)
# Fused draft decode reuses the captured metadata object across draft
# steps. For DCP, build-time host-side decisions such as
+19 -25
View File
@@ -30,7 +30,10 @@ from vllm.config import (
get_current_vllm_config_or_none,
)
from vllm.config.cache import CacheDType
from vllm.distributed.parallel_state import get_dcp_group
from vllm.distributed.parallel_state import (
get_dcp_group,
get_dcp_world_size_and_rank,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -641,6 +644,7 @@ class FlashInferTrtllmAPIDecode:
@dataclass
class FlashInferMetadata:
kv_cache_layout: KVCacheLayout
num_actual_tokens: int
"""Total number of tokens in the batch (excluding padding)."""
@@ -747,17 +751,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self._decode_cudagraph_max_bs,
self.compilation_config.max_cudagraph_capture_size,
)
try:
self.dcp_world_size = get_dcp_group().world_size
self.dcp_rank = get_dcp_group().rank_in_group
self.dcp_kv_cache_interleave_size = (
vllm_config.parallel_config.cp_kv_cache_interleave_size
)
except AssertionError:
# DCP might not be initialized in testing
self.dcp_world_size = 1
self.dcp_rank = 0
self.dcp_kv_cache_interleave_size = 1
self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank(
kv_cache_spec.dcp_sharded
)
self.dcp_kv_cache_interleave_size = (
vllm_config.parallel_config.cp_kv_cache_interleave_size
)
self.use_dcp = self.dcp_world_size > 1
self.dcp_a2a = (
self.use_dcp and vllm_config.parallel_config.dcp_comm_backend == "a2a"
@@ -1424,6 +1423,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# Leave prefill/decode/cascade_wrapper empty, to be populated
# case by case depending on the batch contents and backend selection.
attn_metadata = FlashInferMetadata(
kv_cache_layout=self.kv_cache_layout,
num_actual_tokens=num_actual_tokens,
slot_mapping=common_attn_metadata.slot_mapping,
q_data_type_prefill=self.q_data_type_prefill,
@@ -1904,8 +1904,6 @@ class FlashInferImpl(AttentionImpl):
num_heads, num_kv_heads, is_prefill=False
)
vllm_config = get_current_vllm_config_or_none()
# The layout is resolved after model construction, so read it lazily.
self.cache_config = vllm_config.cache_config if vllm_config else None
# Query pre-quantization needs a single dtype for the whole query tensor.
# SM90 XQA needs BF16/FP16-Q for decode and FP8 for prefill,
# so only enable this for SM100 trtllm-gen where both use FP8-Q.
@@ -1941,11 +1939,6 @@ class FlashInferImpl(AttentionImpl):
else:
self.dcp_combine = partial(cp_lse_ag_out_rs, is_lse_base_on_e=False)
@property
def kv_cache_layout(self) -> KVCacheLayout:
assert self.cache_config is not None
return self.cache_config.get_resolved_kv_cache_layout()
def fused_output_quant_supported(self, quant_key: QuantKey):
if quant_key == kNvfp4Dynamic and self.is_kvcache_nvfp4:
logger.warning_once(
@@ -2111,6 +2104,7 @@ class FlashInferImpl(AttentionImpl):
# performance to make sure it does not introduce any overhead.
num_actual_tokens = attn_metadata.num_actual_tokens
kv_cache_layout = attn_metadata.kv_cache_layout
# FlashInfer treats uint8 KV cache as NVFP4. vLLM stores FP8 KV cache
# as uint8 bytes, so pass FP8 caches with their logical dtype.
@@ -2134,7 +2128,7 @@ class FlashInferImpl(AttentionImpl):
if attn_metadata.use_cascade:
# Cascade attention (rare case).
assert attn_metadata.cascade_wrapper is not None
stride_order = self.kv_cache_layout.layer_view_order
stride_order = kv_cache_layout.layer_view_order
if self.is_kvcache_nvfp4:
kv_cache_views = tuple(
cache.permute(*stride_order)
@@ -2154,7 +2148,7 @@ class FlashInferImpl(AttentionImpl):
num_decode_tokens = attn_metadata.num_decode_tokens
num_prefill_tokens = attn_metadata.num_prefill_tokens
stride_order = self.kv_cache_layout.layer_view_order
stride_order = kv_cache_layout.layer_view_order
kv_cache_permute = kv_cache.permute(*stride_order) # HND and contiguous
# Fix degenerate strides on any size-1 dimension (e.g. num_kv_heads=1
# with TP=8). PyTorch permits non-canonical strides on size-1 dims;
@@ -2311,7 +2305,7 @@ class FlashInferImpl(AttentionImpl):
seq_lens_prefill = attn_metadata.prefill.seq_lens
# This path needs to be enabled with VLLM_KV_CACHE_LAYOUT = HND
assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND"
assert get_flashinfer_layout_string(kv_cache_layout) == "HND"
assert is_strictly_contiguous(prefill_query)
assert is_strictly_contiguous(workspace_buffer)
assert is_strictly_contiguous(block_tables_prefill)
@@ -2501,7 +2495,7 @@ class FlashInferImpl(AttentionImpl):
# trtllm-gen needs HND layout on SM100. XQA is selected
# separately on SM90 and does not use this SM100 layout gate.
if decode_with_trtllm_gen:
assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND"
assert get_flashinfer_layout_string(kv_cache_layout) == "HND"
else:
assert decode_with_xqa
assert is_strictly_contiguous(decode_query)
@@ -2547,7 +2541,7 @@ class FlashInferImpl(AttentionImpl):
window_left=self.window_left,
out=output_padded[:decode_query_tokens],
sinks=self.sinks,
kv_layout=get_flashinfer_layout_string(self.kv_cache_layout),
kv_layout=get_flashinfer_layout_string(kv_cache_layout),
q_len_per_req=q_len_per_req,
mask=attn_metadata.decode.mask,
q_cu_seq_lens=attn_metadata.decode.q_cu_seq_lens,
@@ -2614,7 +2608,7 @@ class FlashInferImpl(AttentionImpl):
sinks=self.sinks,
o_sf_scale=self.o_sf_scale,
out=out,
kv_layout=get_flashinfer_layout_string(self.kv_cache_layout),
kv_layout=get_flashinfer_layout_string(kv_cache_layout),
backend=attn_metadata.decode.kernel.value,
q_len_per_req=q_len_per_req,
max_q_len=max_q_len,
+22 -2
View File
@@ -21,7 +21,12 @@ from vllm.config.cache import _layout_from_name
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor
from vllm.v1.kv_cache_interface import KVCacheLayout, KVCacheSpec, MambaSpec
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheLayout,
KVCacheSpec,
MambaSpec,
)
if TYPE_CHECKING:
from vllm.v1.core.sched.output import SchedulerOutput
@@ -286,9 +291,10 @@ def resolve_kv_cache_layout(
# specs can re-interpret HNC with different sizes as long as the total number of
# bytes is the same. If not block-compact, each spec must agree on HNC to alias
# the same page (this aliasing is done by the Hybrid Memory Allocator, HMA).
kv_cache_specs = tuple(kv_cache_specs or ())
hnc_shapes = {
(spec.num_heads, spec.num_states, spec.page_size_bytes)
for spec in kv_cache_specs or ()
for spec in kv_cache_specs
}
if len(hnc_shapes) > 1:
candidates = [m for m in candidates if m.is_block_compact]
@@ -298,6 +304,20 @@ def resolve_kv_cache_layout(
f"none is in every supported set: {supported_layouts}."
)
dcp_sharding = {
spec.dcp_sharded for spec in kv_cache_specs if isinstance(spec, AttentionSpec)
}
page_sizes = {spec.page_size_bytes for spec in kv_cache_specs}
if len(dcp_sharding) > 1 and len(page_sizes) > 1:
# Sharded target and replicated draft caches need independent groups.
# Block-outer layouts can pack those groups without equalizing pages.
candidates = [layout for layout in candidates if layout.is_block_outermost]
if not candidates:
raise ValueError(
"DCP with a replicated draft and mixed KV page sizes requires "
f"a block-outer KV cache layout; supported sets: {supported_layouts}."
)
if (requested := envs.VLLM_KV_CACHE_LAYOUT) is not None:
layout = _layout_from_name(requested)
if layout not in candidates:
+13 -13
View File
@@ -11,7 +11,6 @@ from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.core.kv_cache_utils import (
BlockHash,
KVCacheBlock,
dcp_world_size_for_kv_cache_spec,
)
from vllm.v1.core.single_type_kv_cache_manager import (
CrossAttentionManager,
@@ -20,6 +19,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
get_manager_for_kv_cache_spec,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheSpec,
@@ -140,9 +140,7 @@ class KVCacheCoordinator(ABC):
role=kv_cache_group.role,
enable_caching=enable_caching,
kv_cache_group_id=i,
dcp_world_size=dcp_world_size_for_kv_cache_spec(
kv_cache_group.kv_cache_spec, dcp_world_size
),
dcp_world_size=dcp_world_size,
pcp_world_size=pcp_world_size,
scheduler_block_size=self.scheduler_block_size,
needs_kv_cache_zeroing=self.kv_cache_config.needs_kv_cache_zeroing,
@@ -664,16 +662,18 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
f"hash_block_size. block_sizes={cacheable_block_sizes}, "
f"hash_block_size={hash_block_size}"
)
assert pcp_world_size == 1, "PCP not support hybrid attn now."
if dcp_world_size > 1:
# DCP shards full-attention KV across ranks and replicates Mamba
# state; other spec types (e.g. sliding window) have no DCP-aware
# handling yet, so reject them explicitly.
for g in kv_cache_config.kv_cache_groups:
assert isinstance(g.kv_cache_spec, (FullAttentionSpec, MambaSpec)), (
for g in kv_cache_config.kv_cache_groups:
spec = g.kv_cache_spec
replicated = not spec.dcp_sharded
if pcp_world_size > 1:
assert isinstance(spec, FullAttentionSpec) or (
isinstance(spec, AttentionSpec) and replicated
), "PCP only supports full attention and replicated draft groups."
if dcp_world_size > 1:
assert isinstance(spec, FullAttentionSpec) or replicated, (
"DCP with hybrid KV cache layouts only supports "
"full-attention and Mamba groups, got: "
f"{type(g.kv_cache_spec).__name__}."
"full-attention, Mamba, and replicated draft groups, got: "
f"{type(spec).__name__}."
)
# Fine-grained hash hits require Mamba "align" and compatible cache
# managers in every group. TP needs hashing finer than the Mamba block;
+19 -41
View File
@@ -682,12 +682,7 @@ def hash_block_tokens(
def resolve_dcp_kv_block_size(spec: KVCacheSpec, dcp_world_size: int) -> int:
"""Return the token span of a cache block under DCP."""
layer_specs = iter_layer_specs(spec)
if len(layer_specs) > 0 and all(
isinstance(layer_spec, AttentionSpec) for layer_spec in layer_specs
):
return spec.block_size * dcp_world_size
return spec.block_size
return spec.block_size * (dcp_world_size if spec.dcp_sharded else 1)
def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCacheSpec:
@@ -707,28 +702,6 @@ def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCache
return replace(spec, block_size=block_size)
def dcp_world_size_for_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> int:
"""Return the DCP size that owns this group's block geometry.
Full-attention KV (including MLA) is sharded across DCP ranks, so prefix
hashing and manager ``block_size`` use the process DCP size. Other specs
keep replicated per-rank state (Mamba, sliding window, chunked-local) and
must keep ``dcp_world_size=1`` even when the process runs with DCP > 1.
Draft MLA groups on the sharded DSpark path are ``FullAttentionSpec`` /
``MLAAttentionSpec`` and therefore keep the process DCP size. A replicated
draft group would need a different spec, not this helper.
"""
if dcp_world_size <= 1:
return 1
inner = spec
if isinstance(spec, UniformTypeKVCacheSpecs):
inner = next(iter(spec.kv_cache_specs.values()))
if isinstance(inner, FullAttentionSpec):
return dcp_world_size
return 1
def resolve_kv_cache_block_sizes(
kv_cache_config: KVCacheConfig,
vllm_config: VllmConfig,
@@ -752,6 +725,8 @@ def resolve_kv_cache_block_sizes(
groups = kv_cache_config.kv_cache_groups
if len(groups) <= 1:
if groups and not groups[0].kv_cache_spec.dcp_sharded:
dcp = 1
bs = cache_config.block_size * dcp
return bs, bs
@@ -1935,18 +1910,21 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]):
kv_cache_spec: The kv cache spec of each attention layer in the model
"""
if is_kv_cache_spec_uniform(
kv_cache_spec
) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec):
return
logger.warning(
"Hybrid KV cache manager is disabled for this hybrid model, "
"This means we do not enable any optimizations for saving KV cache "
"memory (e.g., dropping the KV cache outside the sliding window). "
"The compute of layers like sliding window is still saved."
)
kv_cache_spec.update(_promote_local_kv_cache_specs(kv_cache_spec))
groups: defaultdict[bool, dict[str, KVCacheSpec]] = defaultdict(dict)
for name, spec in kv_cache_spec.items():
replicated = isinstance(spec, AttentionSpec) and not spec.dcp_sharded
groups[replicated][name] = spec
for specs in groups.values():
promoted_specs = _promote_local_kv_cache_specs(specs)
if promoted_specs == specs:
continue
logger.warning(
"Hybrid KV cache manager is disabled for this hybrid model, "
"This means we do not enable any optimizations for saving KV cache "
"memory (e.g., dropping the KV cache outside the sliding window). "
"The compute of layers like sliding window is still saved."
)
kv_cache_spec.update(promoted_specs)
def _approximate_gcd(values: Sequence[int], *, lower_bound: int | None = None) -> int:
@@ -2276,7 +2254,7 @@ def _ensure_min_page_size(
scaled: list[KVCacheGroupSpec] = []
for g in groups:
s = g.kv_cache_spec
kw: dict[str, int] = {"block_size": s.block_size * scale}
kw: dict[str, Any] = {"block_size": s.block_size * scale}
if isinstance(s, (AttentionSpec, MambaSpec)) and s.page_size_padded is not None:
kw["page_size_padded"] = s.page_size_padded * scale
scaled.append(KVCacheGroupSpec(g.layer_names, replace(s, **kw)))
+3 -9
View File
@@ -106,11 +106,9 @@ class SingleTypeKVCacheManager(ABC):
# managers have been validated by the coordinator.
self.cache_hit_alignment_tokens = scheduler_block_size
# The block size for this manager; used for actual block allocation.
self.block_size = kv_cache_spec.block_size
self.dcp_world_size = dcp_world_size
self.pcp_world_size = pcp_world_size
if dcp_world_size > 1:
self.block_size *= dcp_world_size
self.dcp_world_size = dcp_world_size if kv_cache_spec.dcp_sharded else 1
self.pcp_world_size = pcp_world_size if kv_cache_spec.dcp_sharded else 1
self.block_size = kv_cache_spec.block_size * self.dcp_world_size
self.kv_cache_spec = kv_cache_spec
self.block_pool = block_pool
self.enable_caching = enable_caching
@@ -1455,10 +1453,6 @@ class MambaManager(SingleTypeKVCacheManager):
self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs
) -> None:
super().__init__(kv_cache_spec, block_pool, **kwargs)
# Mamba layers use TP instead of DCP, so each rank holds the full
# recurrent state. Undo the DCP/PCP block_size scaling that the base
# class applies for attention groups whose KV cache is partitioned.
self.block_size = kv_cache_spec.block_size
self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode
self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks
self.has_prefill_checkpoint_blocks = (
+24 -5
View File
@@ -158,6 +158,8 @@ class KVCacheSpec:
# number of tokens in a block
block_size: int
dcp_sharded: bool = field(default=False, kw_only=True)
"""Whether DCP shards this cache's token positions across ranks."""
block_stride_alignment: int | None = field(default=None, kw_only=True)
"""Required byte alignment between physical blocks, including packed layers."""
@@ -481,6 +483,7 @@ class HiSparseResidentSpec(KVCacheSpec):
@dataclass(frozen=True, kw_only=True)
class AttentionSpec(KVCacheSpec):
dcp_sharded: bool = True
num_kv_heads: int
head_size: int
dtype: torch.dtype
@@ -536,7 +539,9 @@ class AttentionSpec(KVCacheSpec):
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
parallel_config = vllm_config.parallel_config
kv_shard_count = parallel_config.decode_context_parallel_size
kv_shard_count = (
parallel_config.decode_context_parallel_size if self.dcp_sharded else 1
)
return cdiv(max_len, self.block_size * kv_shard_count)
@@ -568,7 +573,7 @@ class FullAttentionSpec(AttentionSpec):
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
max_model_len = vllm_config.model_config.max_model_len
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size > 1:
if self.dcp_sharded and dcp_world_size > 1:
max_model_len = cdiv(max_model_len, dcp_world_size)
return cdiv(max_model_len, self.block_size) * self.page_size_bytes
@@ -612,6 +617,7 @@ class FullAttentionSpec(AttentionSpec):
head_size_v=specs[0].head_size_v,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
dcp_sharded=specs[0].dcp_sharded,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -698,6 +704,7 @@ class MLAAttentionSpec(FullAttentionSpec):
head_size=specs[0].head_size,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
dcp_sharded=specs[0].dcp_sharded,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -760,6 +767,7 @@ class RSWASpec(FullAttentionSpec):
head_size_v=base.head_size_v,
dtype=base.dtype,
kv_quant_mode=base.kv_quant_mode,
dcp_sharded=base.dcp_sharded,
page_size_padded=base.page_size_padded,
num_head_slots=base.num_head_slots,
state_content_bytes=base.state_content_bytes,
@@ -850,9 +858,10 @@ class SlidingWindowSpec(AttentionSpec):
return cdiv(num_tokens, self.block_size) + 1
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
assert vllm_config.parallel_config.decode_context_parallel_size == 1, (
"DCP not support sliding window."
)
assert (
not self.dcp_sharded
or vllm_config.parallel_config.decode_context_parallel_size == 1
), "DCP not support sliding window."
max_blocks = self.max_admission_blocks_per_request(
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
max_model_len=vllm_config.model_config.max_model_len,
@@ -950,6 +959,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
bounded_replay_set = set(spec.bounded_replay for spec in specs)
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
assert len({spec.dcp_sharded for spec in specs}) == 1
assert (
len(cache_dtype_str_set) == 1
and len(tokens_per_state_set) == 1
@@ -969,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
num_kv_heads=specs[0].num_kv_heads,
head_size=specs[0].head_size,
dtype=specs[0].dtype,
dcp_sharded=specs[0].dcp_sharded,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -1185,6 +1196,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
block_stride_alignment=specs[0].block_stride_alignment,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
dcp_sharded=specs[0].dcp_sharded,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -1217,6 +1229,11 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
kv_cache_specs: dict[str, KVCacheSpec]
def __post_init__(self):
super().__post_init__()
if self.kv_cache_specs:
object.__setattr__(self, "dcp_sharded", self.first_spec.dcp_sharded)
@property
def prefix_cacheable(self) -> bool:
return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values())
@@ -1270,6 +1287,8 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
if len(block_sizes) > 1:
# Different block sizes, not uniform.
return False
if len({spec.dcp_sharded for spec in kv_cache_specs.values()}) > 1:
return False
first_spec = next(iter(kv_cache_specs.values()))
return first_spec.is_uniform_with_collection(kv_cache_specs)
+84 -12
View File
@@ -4,7 +4,7 @@
import contextlib
from collections.abc import Iterable
from dataclasses import dataclass, field, replace
from dataclasses import dataclass, field, fields, replace
from enum import Enum, auto
from typing import TYPE_CHECKING, Any
@@ -27,13 +27,13 @@ from vllm.v1.core.kv_cache_coordinator import (
from vllm.v1.core.kv_cache_utils import (
BlockHashWithGroupId,
ExternalBlockHash,
dcp_world_size_for_kv_cache_spec,
generate_block_hash_extra_keys,
get_block_hash,
get_group_id,
make_block_hash_with_group_id,
maybe_convert_block_hash,
resolve_block_hashes,
resolve_dcp_kv_block_size,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
@@ -48,6 +48,11 @@ from vllm.v1.simple_kv_offload.metadata import (
SimpleCPUOffloadMetadata,
SimpleCPUOffloadWorkerMetadata,
)
from vllm.v1.simple_kv_offload.metrics import (
OUTCOME_TO_FIELD,
MetricName,
SimpleCPUOffloadStats,
)
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
@@ -118,7 +123,8 @@ class BoundaryStoreStats:
symptom is a lower cache hit rate with nothing in the logs, so each decline
reason is counted and exposed through
``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by
``reset()``; not otherwise cleared.
``reset()``, which carries any not-yet-drained deltas; not otherwise
cleared.
"""
published: int = 0
@@ -143,6 +149,7 @@ class SimpleCPUOffloadScheduler:
hash_block_size: int,
lazy_offload: bool = False,
disk_capacity_bytes: int = 0,
use_page_cache: bool = False,
):
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
@@ -169,6 +176,13 @@ class SimpleCPUOffloadScheduler:
self.cpu_kv_cache_config.prefix_cacheable_group_ids
)
self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU
# Must track kv_event_medium.
self._info_labelvalues: tuple[str, ...] = (
"disk" if disk_capacity_bytes > 0 else "cpu",
str(use_page_cache).lower(),
str(lazy_offload).lower(),
str(self.num_cpu_blocks),
)
# Find the full attention kv group for prefix cache matching.
self.fa_gidx = -1
for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups):
@@ -251,6 +265,9 @@ class SimpleCPUOffloadScheduler:
self._load_event_counter: int = 0
self._store_event_counter: int = 0
self.boundary_store_stats = BoundaryStoreStats()
# Interval stats state drained by get_stats()
self._boundary_stats_snapshot = BoundaryStoreStats()
self._interval_load_blocks_completed = 0
# For TP/PP: track partial store completions across steps.
# Events must be reported by all world_size workers before considered complete.
@@ -302,11 +319,7 @@ class SimpleCPUOffloadScheduler:
target = 0
for g in kv_cache_config.prefix_cacheable_groups:
spec = g.kv_cache_spec
# Only full attention is sharded across DCP ranks; replicated specs
# (mamba, sliding window, chunked-local) keep their own block size.
block_size = spec.block_size * dcp_world_size_for_kv_cache_spec(
spec, cp_world_size
)
block_size = resolve_dcp_kv_block_size(spec, cp_world_size)
if isinstance(spec, MambaSpec):
target += 2
elif isinstance(spec, SlidingWindowSpec):
@@ -955,6 +968,42 @@ class SimpleCPUOffloadScheduler:
def get_boundary_store_stats(self) -> BoundaryStoreStats:
return replace(self.boundary_store_stats)
def get_stats(self) -> SimpleCPUOffloadStats:
"""Drain per-step stats for the connector's stats hooks."""
stats = SimpleCPUOffloadStats()
current = self.boundary_store_stats
for outcome, field_name in OUTCOME_TO_FIELD.items():
delta = getattr(current, field_name) - getattr(
self._boundary_stats_snapshot, field_name
)
if delta > 0:
stats.increase_counter(MetricName.SAVE_OUTCOMES, delta, (outcome,))
self._boundary_stats_snapshot = replace(current)
if self._interval_load_blocks_completed:
stats.increase_counter(
MetricName.LOAD_BLOCKS,
self._interval_load_blocks_completed,
)
self._interval_load_blocks_completed = 0
stats.set_gauge(
MetricName.USED_BLOCKS,
self.num_cpu_blocks - self.cpu_block_pool.get_num_free_blocks(),
)
pending = sum(
len(t.cpu_block_ids)
for t in (
*self._store_event_to_blocks.values(),
*self._pending_finished_stores,
*self._abandoned_store_event_to_blocks.values(),
)
)
stats.set_gauge(MetricName.PENDING_STORE_BLOCKS, pending)
stats.set_gauge(MetricName.INFO, 1, self._info_labelvalues)
return stats
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Handle async transfer completions from worker.
@@ -965,7 +1014,9 @@ class SimpleCPUOffloadScheduler:
"""
# --- Load completions ---
for req_id in list(connector_output.finished_recving or []):
self._cleanup_load_request(req_id)
completed_blocks = self._cleanup_load_request(req_id)
if completed_blocks:
self._interval_load_blocks_completed += completed_blocks
# --- Store completions ---
meta = connector_output.kv_connector_worker_meta
@@ -1270,18 +1321,22 @@ class SimpleCPUOffloadScheduler:
if blocks_to_free:
self.cpu_block_pool.free_blocks(blocks_to_free)
def _cleanup_load_request(self, req_id: str) -> None:
def _cleanup_load_request(self, req_id: str) -> int:
"""Release all load resources for a request.
Shared between request_finished() and update_connector_output() paths.
Removes the request from _reqs_to_load, cleans up event mappings,
and frees CPU/GPU touch refs.
Returns the number of blocks in the load if it had been issued to
the worker, else 0 (never-issued loads are not counted as completed
by the caller).
"""
state = self._reqs_to_load.pop(req_id, None)
if state is None:
state = self._abandoned_reqs_to_load.pop(req_id, None)
if state is None:
return
return 0
# Remove from load event mapping (only this req, not whole event)
if state.load_event is not None:
reqs = self._load_event_to_reqs.get(state.load_event)
@@ -1304,6 +1359,10 @@ class SimpleCPUOffloadScheduler:
for bid in state.transfer_meta.gpu_block_ids
)
if state.load_event is None:
return 0
return len(state.transfer_meta.gpu_block_ids)
def _cleanup_store_request(self, req_id: str) -> None:
"""Release store metadata for a request.
@@ -1363,7 +1422,20 @@ class SimpleCPUOffloadScheduler:
if event_idx in self._abandoned_store_event_to_blocks
}
self._cursor = None
self.boundary_store_stats = BoundaryStoreStats()
# Seed the fresh counters with outcomes since the last drain so a
# mid-interval reset() does not drop them from the next report.
undrained = BoundaryStoreStats(
**{
f.name: max(
0,
getattr(self.boundary_store_stats, f.name)
- getattr(self._boundary_stats_snapshot, f.name),
)
for f in fields(BoundaryStoreStats)
}
)
self.boundary_store_stats = undrained
self._boundary_stats_snapshot = BoundaryStoreStats()
# NOTE: _load_event_counter / _store_event_counter are not
# reset as they are monotonic and must stay ahead of the workers
# high-water marks to avoid event index collisions
+284
View File
@@ -0,0 +1,284 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Prometheus metrics for SimpleCPUOffloadConnector.
The stats payload uses the same self-describing wire format as the
offloading connector, so it survives IPC serialization without the
metric classes on the receiving side.
"""
from dataclasses import dataclass
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
class MetricName:
"""Metric names for SimpleCPUOffloadConnector."""
SAVE_OUTCOMES = "vllm:simple_kv_offload_save_outcomes_total"
LOAD_BLOCKS = "vllm:simple_kv_offload_load_blocks_total"
USED_BLOCKS = "vllm:simple_kv_offload_used_blocks"
PENDING_STORE_BLOCKS = "vllm:simple_kv_offload_pending_store_blocks"
INFO = "vllm:simple_kv_offload_info"
# outcome label value -> BoundaryStoreStats field name.
# ``published`` is the sum of all outcomes and is not labeled.
OUTCOME_TO_FIELD = {
"stored": "stored",
"dropped_cpu_full": "dropped_cpu_full",
"dropped_request_gone": "dropped_request_gone",
"dropped_null_block": "dropped_null_block",
"dropped_not_hashed": "dropped_not_hashed",
"skipped_already_cached": "skipped_already_cached",
"skipped_in_flight": "skipped_in_flight",
}
INFO_LABELS = ["backend", "page_cache", "lazy_offload", "capacity_blocks"]
class _MetricType:
"""Type tags embedded in the serialized stats payload."""
COUNTER = "counter"
GAUGE = "gauge"
class _StatsKey:
"""Top-level keys in the serialized stats dict."""
# Maps metric name -> _MetricType value
TYPES = "types"
# Maps metric name -> {label values tuple -> observed value}
DATA = "data"
@dataclass
class SimpleCPUOffloadStats(KVConnectorStats):
"""Stats payload for SimpleCPUOffloadConnector.
The ``data`` dict is structured as::
{
_StatsKey.TYPES: {name: _MetricType.*, ...},
_StatsKey.DATA: {name: {labelvalues: value, ...}, ...},
}
Counter values are summed when aggregating, gauge values keep the
latest snapshot, and unlabeled metrics use ``()`` as their
labelvalues tuple.
"""
def __post_init__(self):
if _StatsKey.DATA not in self.data:
self.reset()
def reset(self):
self.data: dict[str, Any] = {
_StatsKey.TYPES: {},
_StatsKey.DATA: {},
}
@property
def _types(self) -> dict[str, str]:
return self.data[_StatsKey.TYPES]
@property
def _values(self) -> dict[str, Any]:
return self.data[_StatsKey.DATA]
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
if other.is_empty():
return self
assert isinstance(other, SimpleCPUOffloadStats)
for key, other_label_values in other._values.items():
type_str = other._types.get(key)
if type_str is None:
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
self._types.setdefault(key, type_str)
current_label_values = self._values.setdefault(key, {})
for labelvalues, value in other_label_values.items():
if type_str == _MetricType.COUNTER:
assert isinstance(value, int | float)
current_label_values[labelvalues] = (
current_label_values.get(labelvalues, 0) + value
)
elif type_str == _MetricType.GAUGE:
assert isinstance(value, int | float)
current_label_values[labelvalues] = value
else:
raise AssertionError(
f"Unknown metric type '{type_str}' for key: {key}"
)
return self
def reduce(self) -> dict[str, int | float]:
"""Reduce the interval observations to a flat summary for logging."""
return_dict: dict[str, int | float] = {}
for key, label_value_map in self._values.items():
if key == MetricName.INFO:
# Constant config fingerprint; exclude from the log summary.
continue
type_str = self._types.get(key)
if type_str is None:
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
for labelvalues, value in label_value_map.items():
key_with_labels = f"{key}:{labelvalues}" if labelvalues else key
if type_str in (_MetricType.COUNTER, _MetricType.GAUGE):
assert isinstance(value, int | float)
return_dict[key_with_labels] = value
else:
raise AssertionError(
f"Unknown metric type '{type_str}' for key: {key}"
)
return return_dict
def is_empty(self) -> bool:
return not self.data.get(_StatsKey.DATA)
def increase_counter(
self,
counter_name: str,
counter_increase_value: int | float = 1,
labelvalues: tuple[str, ...] = (),
) -> None:
"""Increase a counter on the stats payload."""
self._types.setdefault(counter_name, _MetricType.COUNTER)
counter_values = self._values.setdefault(counter_name, {})
counter_values[labelvalues] = (
counter_values.get(labelvalues, 0) + counter_increase_value
)
def set_gauge(
self,
gauge_name: str,
gauge_value: int | float,
labelvalues: tuple[str, ...] = (),
) -> None:
"""Set a gauge snapshot on the stats payload."""
self._types.setdefault(gauge_name, _MetricType.GAUGE)
gauge_values = self._values.setdefault(gauge_name, {})
gauge_values[labelvalues] = gauge_value
class SimpleCPUOffloadPromMetrics(KVConnectorPromMetrics):
"""Prometheus registration and recording for SimpleCPUOffloadConnector."""
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
# (engine_idx, metric_name, labelvalues) -> bound child metric
self._metrics: dict[tuple[int, str, tuple[str, ...]], PromMetricT] = {}
self._counter_save_outcomes = self._counter_cls(
name="vllm:simple_kv_offload_save_outcomes_total",
documentation=(
"Store admission decisions in SimpleCPUOffloadConnector, by "
"outcome. Outcomes classify eager-mode boundary hand-off "
"stores; lazy-mode stores are not classified."
),
labelnames=self._labelnames + ["outcome"],
)
self._counter_load_blocks = self._counter_cls(
name="vllm:simple_kv_offload_load_blocks_total",
documentation=(
"KV blocks restored to GPU from the offload pool, counted "
"when a load finishes. This is prefill work avoided "
"through offload cache hits."
),
labelnames=self._labelnames,
)
self._gauge_used_blocks = self._gauge_cls(
name="vllm:simple_kv_offload_used_blocks",
documentation=(
"Offload-pool blocks currently pinned by in-flight "
"transfers or cache hits (capacity minus free). Evictable "
"cached blocks are not counted; capacity_blocks is on "
"vllm:simple_kv_offload_info."
),
labelnames=self._labelnames,
multiprocess_mode="mostrecent",
)
self._gauge_pending_store_blocks = self._gauge_cls(
name="vllm:simple_kv_offload_pending_store_blocks",
documentation=(
"Offload blocks in pending stores: queued for dispatch, "
"issued to workers, or awaiting release after a cache "
"reset. A persistently growing value indicates a stuck "
"transfer."
),
labelnames=self._labelnames,
multiprocess_mode="mostrecent",
)
self._gauge_info = self._gauge_cls(
name="vllm:simple_kv_offload_info",
documentation=(
"SimpleCPUOffloadConnector deployment facts. Value is "
"always 1. backend is cpu or disk; page_cache and "
"lazy_offload are true/false; capacity_blocks is the "
"offload pool size."
),
labelnames=self._labelnames + INFO_LABELS,
multiprocess_mode="mostrecent",
)
self._defs: dict[str, PromMetricT] = {
MetricName.SAVE_OUTCOMES: self._counter_save_outcomes,
MetricName.LOAD_BLOCKS: self._counter_load_blocks,
MetricName.USED_BLOCKS: self._gauge_used_blocks,
MetricName.PENDING_STORE_BLOCKS: self._gauge_pending_store_blocks,
MetricName.INFO: self._gauge_info,
}
self._num_label_values: dict[str, int] = {
MetricName.SAVE_OUTCOMES: 1,
MetricName.LOAD_BLOCKS: 0,
MetricName.USED_BLOCKS: 0,
MetricName.PENDING_STORE_BLOCKS: 0,
MetricName.INFO: len(INFO_LABELS),
}
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""Observe transfer statistics."""
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
for key, label_value_map in metric_data.items():
prom_metric = self._defs.get(key)
if prom_metric is None:
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
type_str = metric_types.get(key)
for labelvalues, value in label_value_map.items():
if len(labelvalues) != self._num_label_values[key]:
raise AssertionError(
f"Metric {key} expects "
f"{self._num_label_values[key]} labels, "
f"got {len(labelvalues)}"
)
child = self._metrics.get((engine_idx, key, labelvalues))
if child is None:
engine_labelvalues = self.per_engine_labelvalues[engine_idx]
child = prom_metric.labels(
*(engine_labelvalues + list(labelvalues))
)
self._metrics[(engine_idx, key, labelvalues)] = child
if type_str == _MetricType.COUNTER:
assert isinstance(value, int | float)
child.inc(value)
elif type_str == _MetricType.GAUGE:
assert isinstance(value, int | float)
child.set(value)
else:
raise AssertionError(
f"Unknown metric type '{type_str}' for key: {key}"
)
+8 -1
View File
@@ -81,13 +81,20 @@ class DraftModelProposer(SpecDecodeBaseProposer):
def _create_draft_vllm_config(self) -> VllmConfig:
base = super()._create_draft_vllm_config()
spec = self.speculative_config
target_parallel = base.parallel_config
return replace(
base,
quant_config=None,
parallel_config=replace(
spec.draft_parallel_config,
rank=self.vllm_config.parallel_config.rank,
rank=target_parallel.rank,
decode_context_parallel_size=(
target_parallel.decode_context_parallel_size
),
cp_kv_cache_interleave_size=target_parallel.cp_kv_cache_interleave_size,
dcp_comm_backend=target_parallel.dcp_comm_backend,
dcp_q_replicate=target_parallel.dcp_q_replicate,
),
model_config=spec.draft_model_config,
)
+2
View File
@@ -40,6 +40,8 @@ def check_attention_cp_compatibility(
layer_impl = getattr(layer, "impl", None)
if layer_impl is None:
continue
if not check_pcp and layer_impl.dcp_world_size == 1:
continue
if vllm_config.speculative_config is not None and interleave_size > 1:
assert layer_impl.supports_mtp_with_cp_non_trivial_interleave_size, (
"MTP with cp_kv_cache_interleave_size > 1 is not "
+11
View File
@@ -27,6 +27,7 @@ class BlockTables:
cp_rank: int = 0,
cp_interleave: int = 1,
slot_mapping_enabled: list[bool] | None = None,
dcp_sharded: list[bool] | None = None,
):
self.block_sizes = block_sizes
self.kernel_block_sizes = kernel_block_sizes
@@ -44,6 +45,10 @@ class BlockTables:
slot_mapping_enabled = [True] * self.num_kv_cache_groups
assert len(slot_mapping_enabled) == self.num_kv_cache_groups
self._slot_mapping_enabled = slot_mapping_enabled
if dcp_sharded is None:
dcp_sharded = [True] * self.num_kv_cache_groups
assert len(dcp_sharded) == self.num_kv_cache_groups
self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device)
self.blocks_per_kv_block = [
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
@@ -210,6 +215,7 @@ class BlockTables:
self.block_sizes_tensor,
self.kernel_block_sizes_tensor,
self.slot_mapping_enabled,
self.dcp_sharded,
slot_mappings,
slot_mappings.stride(0),
self.cp_rank,
@@ -283,6 +289,7 @@ def _compute_slot_mappings_kernel(
block_sizes, # [num_kv_cache_groups]
kernel_block_sizes, # [num_kv_cache_groups]
slot_mapping_enabled, # [num_kv_cache_groups]
dcp_sharded, # [num_kv_cache_groups]
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
slot_mappings_stride,
cp_rank,
@@ -312,6 +319,8 @@ def _compute_slot_mappings_kernel(
kv_block_size = tl.load(block_sizes + group_id)
kernel_block_size = tl.load(kernel_block_sizes + group_id)
mapping_enabled = tl.load(slot_mapping_enabled + group_id)
if CP_SIZE != 1:
sharded = tl.load(dcp_sharded + group_id)
req_state_idx = tl.load(idx_mapping + batch_idx)
# idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns
@@ -337,6 +346,8 @@ def _compute_slot_mappings_kernel(
remainder = virtual_block_offsets % CP_INTERLEAVE
local_offsets = rounds * CP_INTERLEAVE + remainder
local_positions = virtual_block_indices * kv_block_size + local_offsets
local_positions = tl.where(sharded, local_positions, positions)
is_local = ~sharded | is_local
block_indices = tl.where(
mapping_enabled, local_positions // kernel_block_size, 0
+14 -1
View File
@@ -21,6 +21,7 @@ import gc
import time
from contextlib import AbstractContextManager
from copy import deepcopy
from dataclasses import replace
from typing import Any, NamedTuple
import numpy as np
@@ -561,7 +562,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return encoder_runner.get_encoder_timing_stats()
def get_kv_cache_spec(self):
return get_kv_cache_spec(self.vllm_config)
kv_cache_spec = get_kv_cache_spec(self.vllm_config)
# A draft running at DCP=1 under a DCP target keeps a replicated cache.
if (
isinstance(self.speculator, DraftModelSpeculator)
and self.speculator.dcp_size == 1
and self.dcp_size > 1
):
for name in self.speculator.draft_attn_layer_names & kv_cache_spec.keys():
kv_cache_spec[name] = replace(kv_cache_spec[name], dcp_sharded=False)
return kv_cache_spec
def initialize_kv_cache(
self,
@@ -587,6 +597,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
block_sizes = []
max_num_blocks_per_group = []
slot_mapping_enabled = []
dcp_sharded = []
for kv_cache_group in kv_cache_config.kv_cache_groups:
spec = kv_cache_group.kv_cache_spec
block_sizes.append(spec.block_size)
@@ -594,6 +605,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec
)
slot_mapping_enabled.append(layer_spec.uses_slot_mapping)
dcp_sharded.append(spec.dcp_sharded)
# Let each cache type account for CP. Attention KV is DCP-sharded,
# while Mamba/GDN recurrent state is replicated across DCP ranks.
max_num_blocks = spec.max_num_blocks_per_req(
@@ -657,6 +669,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
device=self.device,
kernel_block_sizes=self.kernel_block_sizes,
slot_mapping_enabled=slot_mapping_enabled,
dcp_sharded=dcp_sharded,
cp_size=self.dcp_size,
cp_rank=self.dcp_rank,
cp_interleave=self.cp_interleave,
+4 -1
View File
@@ -145,7 +145,10 @@ class PCPManager:
if speculative_config is not None:
if speculative_config.use_dspark():
dcp_size = parallel_config.decode_context_parallel_size
if dcp_size not in (1, pcp_size):
if (
dcp_size not in (1, pcp_size)
and speculative_config.draft_model_config.use_mla
):
raise NotImplementedError(
"MRV2 PCP DSpark requires DCP=1 or DCP=PCP; got "
f"DCP={dcp_size}, PCP={pcp_size}."
@@ -37,12 +37,18 @@ class DFlashSpeculator(DraftModelSpeculator):
def __init__(self, vllm_config: VllmConfig, device: torch.device):
parallel_config = vllm_config.parallel_config
if parallel_config.prefill_context_parallel_size > 1:
vllm_config = copy.copy(vllm_config)
vllm_config.parallel_config = replace(
parallel_config,
prefill_context_parallel_size=1,
)
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
vllm_config = copy.copy(vllm_config)
vllm_config.parallel_config = replace(
parallel_config,
prefill_context_parallel_size=1,
decode_context_parallel_size=(
parallel_config.decode_context_parallel_size
if speculative_config.draft_model_config.use_mla
else 1
),
)
super().__init__(vllm_config, device)
self.hidden_states = torch.zeros(
@@ -427,7 +433,7 @@ class DFlashSpeculator(DraftModelSpeculator):
self.block_tables.input_block_tables[gid],
self.block_tables.kernel_block_sizes[gid],
self.block_tables.cp_rank,
self.block_tables.cp_size,
self.dcp_size,
self.block_tables.cp_interleave,
self.parallel_drafting_token_id,
self.num_query_per_req,
+3 -2
View File
@@ -138,6 +138,7 @@ class DraftModelSpeculator(BaseSpeculator):
# DP configuration
self.dp_size = vllm_config.parallel_config.data_parallel_size
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size
self.eplb_state: EplbState | None = None
@@ -332,14 +333,14 @@ class DraftModelSpeculator(BaseSpeculator):
out=draft_seq_lens_cpu_upper_bound[:num_reqs],
)
draft_seq_lens_cpu_upper_bound[:num_reqs].clamp_(max=self.max_model_len)
if dcp_local_seq_lens is None and self.block_tables.cp_size > 1:
if dcp_local_seq_lens is None and self.dcp_size > 1:
# Draft steps advance and rewind their own global sequence lengths,
# so the target model's DCP-local lengths may already be stale.
dcp_local_seq_lens = prepare_dcp_local_seq_lens(
self.input_buffers.dcp_local_seq_lens,
self.input_buffers.seq_lens,
num_reqs,
self.block_tables.cp_size,
self.dcp_size,
self.block_tables.cp_rank,
self.block_tables.cp_interleave,
)
+1
View File
@@ -817,6 +817,7 @@ class InputBatch:
self.allowed_token_ids_mask_cpu_tensor[empty_index] = (
self.allowed_token_ids_mask_cpu_tensor[last_req_index]
)
self.allowed_token_ids_mask_cpu_tensor[last_req_index].fill_(False)
bad_words_token_ids = self.bad_words_token_ids.pop(last_req_index, None)
if bad_words_token_ids is not None:
+12 -3
View File
@@ -801,11 +801,20 @@ class Worker(WorkerBase):
# related to kv cache connector (e.g. kv cache sharing layers).
ensure_kv_transfer_initialized(self.vllm_config, kv_cache_config)
# If the connector provides a custom memory pool (e.g. Mooncake
# NVLink/BAREX), use it for KV cache allocation; otherwise fall
# back to the standard CuMem pool.
mem_pool_context = (
get_kv_transfer_group().get_mem_pool_context()
if has_kv_transfer_group()
else None
)
if mem_pool_context is None:
mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache")
self.model_runner.initialize_kv_cache(
kv_cache_config,
kv_cache_allocation_context=self._maybe_get_memory_pool_context(
tag="kv_cache"
),
kv_cache_allocation_context=mem_pool_context,
)
# Build KV-zero metadata outside the CuMem pool so the bookkeeping