Merge branch 'main' into gdn-recoverssm
This commit is contained in:
@@ -13,7 +13,7 @@ else()
|
||||
FetchContent_Declare(
|
||||
flashkda
|
||||
GIT_REPOSITORY https://github.com/vllm-project/FlashKDA.git
|
||||
GIT_TAG b59532f1f464fbd536272780e30df5bf6a2ccc02
|
||||
GIT_TAG 17a037d98da546deb4591e967cf961a43c034d8b
|
||||
GIT_PROGRESS TRUE
|
||||
GIT_SUBMODULES cutlass
|
||||
)
|
||||
|
||||
@@ -144,7 +144,7 @@ When batch invariance is enabled, vLLM:
|
||||
|
||||
1. Uses deterministic kernel implementations for attention and other operations
|
||||
2. Ensures consistent numerical behavior across different batch sizes
|
||||
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode)
|
||||
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant)
|
||||
4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
|
||||
runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out
|
||||
(not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes).
|
||||
|
||||
@@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [
|
||||
"path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py",
|
||||
"key": "metrics-nixl",
|
||||
},
|
||||
{
|
||||
"path": "vllm/v1/simple_kv_offload/metrics.py",
|
||||
"key": "metrics-simple-kv-offload",
|
||||
},
|
||||
{"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"},
|
||||
]
|
||||
|
||||
|
||||
@@ -45,6 +45,30 @@ The following metrics are exposed:
|
||||
|
||||
--8<-- "gen:metrics-nixl"
|
||||
|
||||
## Simple CPU Offload Connector Metrics
|
||||
|
||||
These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector
|
||||
is configured (e.g. `--kv-transfer-config='{"kv_connector":
|
||||
"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config":
|
||||
{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are
|
||||
updated once per engine step.
|
||||
|
||||
Caveats to keep in mind when interpreting them:
|
||||
|
||||
- A "completed" store means the write syscalls returned; it is **not**
|
||||
fsync-durable, and the disk backend's file is process-lifetime scratch,
|
||||
unlinked at startup and shutdown.
|
||||
- `save_outcomes_total` classifies eager-mode boundary hand-off stores;
|
||||
lazy-mode stores are not classified.
|
||||
- `used_blocks` counts blocks pinned by in-flight transfers or cache hits;
|
||||
warm cached blocks that are evictable are not counted. Use the
|
||||
`capacity_blocks` label of `simple_kv_offload_info` as the denominator.
|
||||
- Counters and gauges are quantized to engine steps. They are reported by the
|
||||
scheduler process and reflect engine-wide logical block counts, not values
|
||||
pooled from individual tensor-parallel workers.
|
||||
|
||||
--8<-- "gen:metrics-simple-kv-offload"
|
||||
|
||||
## Model Flops Utilization (MFU) Performance Metrics
|
||||
|
||||
These metrics are available via `--enable-mfu-metrics`:
|
||||
|
||||
@@ -566,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
|
||||
assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_syncs_aiter_a4w4_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.delenv("AITER_SITUV2_A8W4", raising=False)
|
||||
mp.delenv("AITER_SITUV2_A4W4", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
|
||||
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
|
||||
assert "AITER_SITUV2_A8W4" not in os.environ
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.delenv("AITER_SITUV2_A8W4", raising=False)
|
||||
mp.delenv("AITER_SITUV2_A4W4", raising=False)
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
|
||||
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
|
||||
assert "AITER_SITUV2_A8W4" not in os.environ
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_clears_aiter_env_when_disabled(
|
||||
@pytest.mark.parametrize(
|
||||
"value,expected_act,expected_env",
|
||||
[
|
||||
# Unset / auto / legacy "1" all mean the a4w4 default.
|
||||
(None, "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("0", "a16w4", {}),
|
||||
("a16w4", "a16w4", {}),
|
||||
("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}),
|
||||
],
|
||||
)
|
||||
def test_aiter_moe_situv2_activation_syncs_aiter_env(
|
||||
value: str | None,
|
||||
expected_act: str,
|
||||
expected_env: dict[str, str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Each SiTUv2 activation choice sets exactly its AITER dispatch env."""
|
||||
import os
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
# Stale AITER overrides must be replaced by the vLLM selection.
|
||||
mp.setenv("AITER_SITUV2_A8W4", "1")
|
||||
mp.setenv("AITER_SITUV2_A4W4", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0")
|
||||
if value is None:
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
|
||||
else:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value)
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act
|
||||
assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is (
|
||||
expected_act == "a8w4"
|
||||
)
|
||||
assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4")
|
||||
for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"):
|
||||
assert os.environ.get(name) == expected_env.get(name)
|
||||
|
||||
assert "AITER_SITUV2_A4W4" not in os.environ
|
||||
|
||||
def test_aiter_moe_situv2_rejects_unknown_activation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8")
|
||||
_reload_envs()
|
||||
import vllm.envs as envs
|
||||
|
||||
with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"):
|
||||
_ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_padding", [True, False])
|
||||
|
||||
@@ -21,7 +21,12 @@ Usage:
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
deepgemm_post_process_fp8_weight_block,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import fp8_einsum
|
||||
|
||||
# -- Default dimensions matching DeepSeek V3/V4 --------------------------
|
||||
HEAD_DIM = 512
|
||||
@@ -33,6 +38,49 @@ FP8_DTYPE = torch.float8_e4m3fn
|
||||
EPS = 1e-10
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103"
|
||||
)
|
||||
@pytest.mark.parametrize("groups", [2, 4])
|
||||
@pytest.mark.parametrize("tokens", [1, 6, 18, 32])
|
||||
def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens):
|
||||
"""Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize."""
|
||||
from flashinfer import mxfp8_quantize
|
||||
|
||||
from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL
|
||||
|
||||
torch.manual_seed(0)
|
||||
n = groups * 1024
|
||||
x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128]
|
||||
positions = torch.randint(0, 128, (tokens,), device="cuda")
|
||||
rope = make_cos_sin_cache(128, device="cuda")
|
||||
wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE)
|
||||
scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float())
|
||||
wq, ws = deepgemm_post_process_fp8_weight_block(
|
||||
wq, scale, (1, 32), False, True, groups
|
||||
)
|
||||
q, sf = _FUSED_WO_A_KERNEL(
|
||||
x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws
|
||||
)
|
||||
|
||||
a, sa = fused_inv_rope_fp8_quant(
|
||||
x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True
|
||||
)
|
||||
y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16)
|
||||
fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32))
|
||||
ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl")
|
||||
|
||||
def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks]
|
||||
return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1)
|
||||
|
||||
actual, expected = (
|
||||
d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0)
|
||||
for d, s in ((q, sf), (ref, ref_sf))
|
||||
)
|
||||
assert (actual - expected).norm() / expected.norm() < 3e-3
|
||||
assert not rows(sf)[tokens:].any(), "padded scale rows must be zero"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Helpers
|
||||
# =========================================================================
|
||||
|
||||
@@ -942,8 +942,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
|
||||
impl.o_sf_scale = None
|
||||
impl.window_left = -1
|
||||
impl.sinks = None
|
||||
impl.cache_config = unittest.mock.Mock()
|
||||
impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC
|
||||
|
||||
layer = unittest.mock.Mock(
|
||||
_q_scale=torch.tensor(1.0),
|
||||
@@ -959,6 +957,7 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
|
||||
q_len_per_req=1,
|
||||
)
|
||||
attn_metadata = flashinfer_backend.FlashInferMetadata(
|
||||
kv_cache_layout=KVCacheLayout.LBHNC,
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.empty(0, dtype=torch.int64),
|
||||
q_data_type_prefill=torch.bfloat16,
|
||||
|
||||
@@ -101,6 +101,7 @@ def test_flash_attention_geometry_comes_from_the_group():
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
dcp_sharded=True,
|
||||
)
|
||||
|
||||
with (
|
||||
|
||||
@@ -1599,19 +1599,142 @@ def test_project_kv_cache_groups_to_worker():
|
||||
assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"}
|
||||
|
||||
|
||||
def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only():
|
||||
dcp = 8
|
||||
full = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
@pytest.mark.parametrize("sliding_window", [None, 256])
|
||||
@pytest.mark.parametrize("disable_hybrid", [False, True])
|
||||
@pytest.mark.parametrize("pcp_size", [1, 4])
|
||||
def test_dcp_target_allocates_replicated_draft_independently(
|
||||
monkeypatch, sliding_window, disable_hybrid, pcp_size
|
||||
):
|
||||
"""A draft must retain all positions even when the target shards them."""
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
|
||||
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
config.parallel_config.prefill_context_parallel_size = pcp_size
|
||||
config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hybrid
|
||||
config.cache_config.block_size = 16
|
||||
config.cache_config.kv_cache_layout = None
|
||||
draft_args = dict(
|
||||
block_size=16,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
mla = new_mla_spec()
|
||||
mamba = new_mamba_spec()
|
||||
uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla})
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1
|
||||
draft = (
|
||||
FullAttentionSpec(**draft_args)
|
||||
if sliding_window is None
|
||||
else SlidingWindowSpec(**draft_args, sliding_window=sliding_window)
|
||||
)
|
||||
specs = {"target": new_mla_spec(), "draft": draft}
|
||||
layout = resolve_kv_cache_layout(config, [["LBHNC", "BLHNC"]], specs.values())
|
||||
assert layout == KVCacheLayout.BLHNC
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
assert [group.layer_names for group in groups] == [["target"], ["draft"]]
|
||||
widths = [g.kv_cache_spec.max_num_blocks_per_req(config, 1024) for g in groups]
|
||||
assert widths == [16, 64]
|
||||
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=16 * 1024 * 1024
|
||||
)
|
||||
scheduler_config = generate_scheduler_kv_cache_config([cache_config])
|
||||
assert get_max_concurrency_for_kv_cache_config(config, cache_config) > 0
|
||||
assert get_max_concurrency_for_kv_cache_config(config, cache_config) == (
|
||||
get_max_concurrency_for_kv_cache_config(config, scheduler_config)
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
scheduler_config,
|
||||
max_model_len=1024,
|
||||
hash_block_size=16,
|
||||
scheduler_block_size=64,
|
||||
dcp_world_size=4,
|
||||
pcp_world_size=pcp_size,
|
||||
enable_caching=True,
|
||||
)
|
||||
assert manager.coordinator.group_block_sizes == (64, 16)
|
||||
request = make_request("replicated-draft", [1] * 65, block_size=16, hash_fn=sha256)
|
||||
blocks = manager.allocate_slots(request, 65)
|
||||
assert blocks is not None
|
||||
assert [len(group) for group in blocks.blocks] == [2, 5]
|
||||
manager.cache_blocks(request, 64)
|
||||
cached_request = make_request(
|
||||
"cached-draft", [1] * 65, block_size=16, hash_fn=sha256
|
||||
)
|
||||
cached_blocks, num_cached, _ = manager.get_computed_blocks(cached_request)
|
||||
assert num_cached == 64
|
||||
assert [len(group) for group in cached_blocks.blocks] == [1, 4]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_mla", [False, True])
|
||||
def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
draft = replace(
|
||||
new_mla_spec() if use_mla else new_kv_cache_spec(),
|
||||
block_size=16,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
merged = type(draft).merge([draft, draft])
|
||||
wrapped = UniformTypeKVCacheSpecs.from_specs({"draft": merged})
|
||||
assert wrapped is not None and not wrapped.dcp_sharded
|
||||
assert merged.max_num_blocks_per_req(config, 1024) == 64
|
||||
assert merged.max_memory_usage_bytes(config) == 64 * draft.page_size_bytes
|
||||
assert kv_cache_utils.resolve_dcp_kv_block_size(merged, 4) == 16
|
||||
with pytest.raises(AssertionError):
|
||||
type(draft).merge([draft, replace(draft, dcp_sharded=True)])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("with_draft", [False, True])
|
||||
@pytest.mark.parametrize("indexer_alignment", [None, 512])
|
||||
def test_sparse_mla_preserves_physical_row_addressing(with_draft, indexer_alignment):
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
common = dict(block_size=64, num_kv_heads=1, dtype=torch.uint8)
|
||||
specs = {
|
||||
"target": MLAAttentionSpec(
|
||||
**common,
|
||||
head_size=576,
|
||||
state_content_bytes=656,
|
||||
block_stride_alignment=656,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
is_index_group_leader=True,
|
||||
),
|
||||
"indexer": MLAAttentionSpec(
|
||||
**common,
|
||||
head_size=132,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
block_stride_alignment=indexer_alignment,
|
||||
),
|
||||
"draft": SlidingWindowSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=64,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=2048,
|
||||
dcp_sharded=False,
|
||||
),
|
||||
}
|
||||
if not with_draft:
|
||||
del specs["draft"]
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=8 * 1024 * 1024
|
||||
)
|
||||
caches = allocate_kv_cache(
|
||||
cache_config, torch.device("cpu"), KVCacheLayout.BLHNC, [64] * len(groups)
|
||||
)
|
||||
cache = caches["target"].squeeze(1)
|
||||
if indexer_alignment is not None:
|
||||
assert caches["indexer"].stride(0) % indexer_alignment == 0
|
||||
rows, stride_rows = flat_kv_row_view(cache, 64)
|
||||
cache[1, 0].fill_(7)
|
||||
torch.testing.assert_close(rows[stride_rows], cache[1, 0])
|
||||
assert torch.all(rows[stride_rows] == 7)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -3815,6 +3938,8 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
"layer_1": before_spec_1,
|
||||
"layer_2": before_spec_2,
|
||||
}
|
||||
kv_cache_spec["draft_layer_1"] = replace(before_spec_1, dcp_sharded=False)
|
||||
kv_cache_spec["draft_layer_2"] = replace(before_spec_2, dcp_sharded=False)
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
expected_spec_1 = new_kv_cache_spec(block_size=64)
|
||||
expected_spec_2 = new_kv_cache_spec(
|
||||
@@ -3823,6 +3948,7 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
assert kv_cache_spec["layer_1"] == expected_spec_1
|
||||
assert kv_cache_spec["layer_2"] == expected_spec_2
|
||||
assert kv_cache_spec["layer_2"].page_size_bytes == 64 * 1024
|
||||
assert kv_cache_spec["draft_layer_2"] == replace(expected_spec_2, dcp_sharded=False)
|
||||
|
||||
# 2. has_full_attention and has_chunked_local_attention
|
||||
before_spec_1 = new_kv_cache_spec()
|
||||
@@ -3874,6 +4000,11 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
with pytest.raises(ValueError):
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
|
||||
# Replicated Mamba state still requires the hybrid cache manager.
|
||||
kv_cache_spec = {"attention": new_kv_cache_spec(), "mamba": new_mamba_spec()}
|
||||
with pytest.raises(ValueError):
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
|
||||
|
||||
def test_unify_kv_cache_spec_page_size_mamba():
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/43626.
|
||||
|
||||
@@ -1336,12 +1336,12 @@ def make_kv_cache_config_three_types(
|
||||
)
|
||||
|
||||
|
||||
def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
@pytest.mark.parametrize("draft_sharded", [True, False])
|
||||
def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
|
||||
"""Prefix lookup must use each group's DCP size, not the process-wide one.
|
||||
|
||||
Target and draft MLA are both sharded (DCP=8); Mamba stays replicated
|
||||
(DCP=1). Hits then align to the sharded full-attention block, not the
|
||||
unsharded page size.
|
||||
Target MLA is sharded; draft MLA can opt out and Mamba stays replicated.
|
||||
Hits must align to the target block in either case.
|
||||
"""
|
||||
block_size = 16
|
||||
dcp = 8
|
||||
@@ -1362,6 +1362,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
KVCacheGroupSpec(
|
||||
["draft_mla"],
|
||||
MLAAttentionSpec(
|
||||
dcp_sharded=draft_sharded,
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
@@ -1388,10 +1389,10 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
)
|
||||
target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers
|
||||
assert target_mgr.dcp_world_size == dcp
|
||||
assert draft_mgr.dcp_world_size == dcp
|
||||
assert draft_mgr.dcp_world_size == (dcp if draft_sharded else 1)
|
||||
assert mamba_mgr.dcp_world_size == 1
|
||||
assert target_mgr.block_size == sharded_block
|
||||
assert draft_mgr.block_size == sharded_block
|
||||
assert draft_mgr.block_size == (sharded_block if draft_sharded else block_size)
|
||||
assert mamba_mgr.block_size == block_size
|
||||
|
||||
hash_fn = sha256
|
||||
@@ -1407,7 +1408,11 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 2 * sharded_block
|
||||
assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16]
|
||||
assert [len(group) for group in computed_blocks.blocks] == [
|
||||
2,
|
||||
2 if draft_sharded else 16,
|
||||
16,
|
||||
]
|
||||
|
||||
manager.free(req0)
|
||||
manager.free(req1)
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.config import CompilationConfig, VllmConfig, replace
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
|
||||
from ..utils import (
|
||||
Messages,
|
||||
@@ -196,6 +197,38 @@ def test_draft_model_engine_args_tensor_parallelism():
|
||||
assert draft_config.quant_config is None
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("dcp_size", [1, 2])
|
||||
def test_draft_model_inherits_target_dcp_config(monkeypatch, dcp_size):
|
||||
monkeypatch.setattr(current_platform, "device_count", lambda: 16)
|
||||
target_config = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=16,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=16,
|
||||
dcp_comm_backend="ag_rs",
|
||||
distributed_executor_backend="mp",
|
||||
speculative_config={
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"method": "draft_model",
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
).create_engine_config()
|
||||
proposer = object.__new__(DraftModelProposer)
|
||||
proposer.vllm_config = target_config
|
||||
proposer.speculative_config = target_config.speculative_config
|
||||
draft_config = proposer._create_draft_vllm_config()
|
||||
draft_parallel = draft_config.parallel_config
|
||||
assert draft_parallel.decode_context_parallel_size == dcp_size
|
||||
assert draft_parallel.cp_kv_cache_interleave_size == 16
|
||||
assert draft_parallel.dcp_comm_backend == "ag_rs"
|
||||
assert (
|
||||
draft_parallel.dcp_q_replicate == target_config.parallel_config.dcp_q_replicate
|
||||
)
|
||||
assert draft_parallel is not target_config.parallel_config
|
||||
assert target_config.parallel_config.decode_context_parallel_size == dcp_size
|
||||
|
||||
|
||||
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
|
||||
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
|
||||
so we can test it without instantiating a full proposer."""
|
||||
|
||||
@@ -500,7 +500,7 @@ def test_decode_bench_connector_skips_null_padding():
|
||||
|
||||
|
||||
def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
|
||||
"""DCP scales full-attention blocks but not sliding-window blocks."""
|
||||
"""DCP scales target blocks but not the replicated draft's window."""
|
||||
block_size = 16
|
||||
dcp_world_size = 2
|
||||
vllm_config = create_vllm_config(
|
||||
@@ -515,6 +515,7 @@ def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
|
||||
dtype=torch.float32,
|
||||
)
|
||||
sliding_window = SlidingWindowSpec(
|
||||
dcp_sharded=False,
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
|
||||
@@ -290,6 +290,8 @@ class _FakeKVTransferConfig:
|
||||
class _FakeModelConfig:
|
||||
model = "test-model"
|
||||
use_mla = False
|
||||
enable_sleep_mode = False
|
||||
enable_cumem_allocator = False
|
||||
|
||||
def get_num_layers(self, parallel_config) -> int:
|
||||
return 1
|
||||
@@ -432,6 +434,64 @@ def _patch_worker_runtime(
|
||||
monkeypatch.setattr(worker, "LookupKeyServer", MagicMock())
|
||||
|
||||
|
||||
def test_create_mem_pool_returns_none_when_unconfigured():
|
||||
assert worker.MooncakeStoreWorker._create_mem_pool({}, _FakeModelConfig()) is None
|
||||
|
||||
|
||||
def test_create_mem_pool_rejects_unknown_pool():
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Unsupported custom_mem_pool='UNKNOWN'",
|
||||
):
|
||||
worker.MooncakeStoreWorker._create_mem_pool(
|
||||
{"custom_mem_pool": "unknown"}, _FakeModelConfig()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"conflicting_option", ["enable_sleep_mode", "enable_cumem_allocator"]
|
||||
)
|
||||
def test_create_mem_pool_rejects_cumem_conflicts(conflicting_option):
|
||||
model_config = _FakeModelConfig()
|
||||
setattr(model_config, conflicting_option, True)
|
||||
|
||||
with pytest.raises(ValueError, match="custom_mem_pool is incompatible"):
|
||||
worker.MooncakeStoreWorker._create_mem_pool(
|
||||
{"custom_mem_pool": "NVLINK"}, model_config
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pool_type", "allocator_name"),
|
||||
[("nvlink", "NVLinkAllocator"), ("barex", "BarexAllocator")],
|
||||
)
|
||||
def test_create_mem_pool_uses_mooncake_allocator(
|
||||
monkeypatch, pool_type, allocator_name
|
||||
):
|
||||
allocator_cls = MagicMock()
|
||||
allocator = allocator_cls.get_allocator.return_value
|
||||
allocator_module = types.ModuleType("mooncake.allocator")
|
||||
setattr(allocator_module, allocator_name, allocator_cls)
|
||||
mooncake_module = types.ModuleType("mooncake")
|
||||
mooncake_module.allocator = allocator_module # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, "mooncake", mooncake_module)
|
||||
monkeypatch.setitem(sys.modules, "mooncake.allocator", allocator_module)
|
||||
|
||||
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 2)
|
||||
mem_pool = MagicMock()
|
||||
mem_pool_ctor = MagicMock(return_value=mem_pool)
|
||||
monkeypatch.setattr(torch.cuda, "MemPool", mem_pool_ctor)
|
||||
|
||||
result = worker.MooncakeStoreWorker._create_mem_pool(
|
||||
{"custom_mem_pool": pool_type}, _FakeModelConfig()
|
||||
)
|
||||
|
||||
assert result is mem_pool
|
||||
allocator_cls.get_allocator.assert_called_once_with(torch.device("cuda", 2))
|
||||
allocator.allocator.assert_called_once_with()
|
||||
mem_pool_ctor.assert_called_once_with(allocator.allocator.return_value)
|
||||
|
||||
|
||||
def test_pool_key_to_string_without_prefix_is_unchanged():
|
||||
"""Default (empty) cache_prefix keeps keys byte-identical to the
|
||||
historical unprefixed format so existing deployments keep their hits."""
|
||||
|
||||
@@ -68,6 +68,7 @@ class MockConnector(KVConnectorBase_V1):
|
||||
# Override just build_kv_connector_stats
|
||||
mock.build_kv_connector_stats = cls.build_kv_connector_stats
|
||||
mock.get_kv_connector_stats.return_value = None
|
||||
mock.get_mem_pool_context.return_value = None
|
||||
return mock
|
||||
|
||||
@classmethod
|
||||
@@ -201,6 +202,36 @@ def mc() -> MultiConnector:
|
||||
return mc
|
||||
|
||||
|
||||
def test_multi_connector_mem_pool_context_none(mc: MultiConnector):
|
||||
assert mc.get_mem_pool_context() is None
|
||||
|
||||
|
||||
def test_multi_connector_forwards_mem_pool_context(mc: MultiConnector):
|
||||
context = MagicMock()
|
||||
provider = MagicMock(spec_set=KVConnectorBase_V1)
|
||||
provider.get_mem_pool_context.return_value = context
|
||||
mc._connectors = [mc._connectors[0], provider]
|
||||
|
||||
assert mc.get_mem_pool_context() is context
|
||||
provider.get_mem_pool_context.assert_called_once_with()
|
||||
|
||||
|
||||
def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector):
|
||||
providers = [
|
||||
MagicMock(spec_set=KVConnectorBase_V1),
|
||||
MagicMock(spec_set=KVConnectorBase_V1),
|
||||
]
|
||||
for provider in providers:
|
||||
provider.get_mem_pool_context.return_value = MagicMock()
|
||||
mc._connectors = providers
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Multiple connectors provide a KV cache memory pool",
|
||||
):
|
||||
mc.get_mem_pool_context()
|
||||
|
||||
|
||||
# Helper function to compare directories recursively
|
||||
def _compare_directories(dir1: Path, dir2: Path) -> bool:
|
||||
"""Compares two directories recursively for identical content."""
|
||||
@@ -320,6 +351,7 @@ def test_multi_example_connector_consistency():
|
||||
# First three events are from initialization. Layer hooks run before the
|
||||
# deferred load starts after the forward pass.
|
||||
expected_worker_prefix = [
|
||||
"get_mem_pool_context",
|
||||
"register_kv_caches",
|
||||
"set_host_xfer_buffer_ops",
|
||||
"get_handshake_metadata",
|
||||
|
||||
@@ -7,6 +7,7 @@ import os
|
||||
import queue
|
||||
import tempfile
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
@@ -49,6 +50,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
|
||||
NixlKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
RemoteMeta,
|
||||
ReqMeta,
|
||||
compute_nixl_compatibility_hash,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_transfer_state import (
|
||||
@@ -2087,6 +2090,10 @@ def recv_worker():
|
||||
worker._recving_transfers = defaultdict(list)
|
||||
worker._failed_recv_reqs = queue.Queue()
|
||||
worker._recv_failures = set()
|
||||
worker._handshake_lock = threading.RLock()
|
||||
worker._handshake_futures = {}
|
||||
worker._remote_agents = {}
|
||||
worker._engine_by_address = {}
|
||||
worker._replicated_pcp_done_sending = set()
|
||||
worker._invalid_block_ids = queue.Queue()
|
||||
worker._pending_recv_notifs = {}
|
||||
@@ -2681,6 +2688,174 @@ def test_engine_ttl_disabled(default_vllm_config, dist_init):
|
||||
assert engine_id in worker.dst_xfer_side_handles
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
class TestPeerReplacement:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup(self, monkeypatch):
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0})
|
||||
config.kv_transfer_config.kv_buffer_device = "cpu"
|
||||
platform = SimpleNamespace(
|
||||
device_type="cpu",
|
||||
discover_numa_topology=lambda: [],
|
||||
get_nixl_memory_type=lambda: "DRAM",
|
||||
is_rocm=lambda: False,
|
||||
)
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper", FakeNixlWrapper),
|
||||
patch.object(bw, "current_platform", platform),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(
|
||||
bw, "get_current_attn_backends", return_value=[FlashAttentionBackend]
|
||||
),
|
||||
):
|
||||
self.worker = FakeNixlConnectorWorker(config, "local", hand_shake_latency=0)
|
||||
self.nixl = self.worker.nixl_wrapper
|
||||
self.transport = MagicMock(wraps=self.nixl)
|
||||
self.worker.nixl_wrapper = self.transport
|
||||
monkeypatch.setattr(
|
||||
self.worker._handshake_initiation_executor,
|
||||
"submit",
|
||||
lambda *args: Future(),
|
||||
)
|
||||
self._connect("old")
|
||||
yield
|
||||
self.worker.shutdown()
|
||||
|
||||
def _connect(self, engine_id="new", host="localhost", port=1234):
|
||||
future = self.worker._ensure_handshake(engine_id, host, port, 1)
|
||||
self.worker.REMOTE_ENGINE_ID = engine_id
|
||||
self.nixl.REMOTE_AGENT_NAME = engine_id
|
||||
future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id))
|
||||
|
||||
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True):
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta(
|
||||
local_block_ids=(blocks,),
|
||||
local_physical_block_ids=(blocks,),
|
||||
tp_size=1,
|
||||
remote=RemoteMeta(
|
||||
([2],), "localhost", 1234, engine_id, f"{engine_id}-prefill"
|
||||
),
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
return metadata
|
||||
|
||||
def test_cleans_old_peer_after_confirmed_handshake(self):
|
||||
self._connect("healthy", host="other-host")
|
||||
old_handle = self.worker.dst_xfer_side_handles["old"][0]
|
||||
self.worker.start_load_kv(self._request())
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
assert set(self.worker._remote_agents) == {"new", "healthy"}
|
||||
assert self.worker._engine_by_address == {
|
||||
("localhost", 1234): "new",
|
||||
("other-host", 1234): "healthy",
|
||||
}
|
||||
with pytest.raises(KeyError):
|
||||
self.worker.transfer_topo.get_engine_info("old")
|
||||
|
||||
self.worker.start_load_kv(NixlConnectorMetadata())
|
||||
assert self.worker.get_transfer_results().finished_recving == {"new-req"}
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
@pytest.mark.parametrize("host,port", [("other-host", 1234), ("localhost", 5678)])
|
||||
def test_preserves_other_addresses(self, host, port):
|
||||
self._connect(host=host, port=port)
|
||||
self.worker.get_transfer_results()
|
||||
assert "old" in self.worker._remote_agents
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
def test_preserves_old_peer_when_handshake_fails(self):
|
||||
self.worker.start_load_kv(self._request())
|
||||
self.worker._handshake_futures["new"].set_exception(
|
||||
RuntimeError("handshake failed")
|
||||
)
|
||||
assert self.worker.get_transfer_results().failed_recving == {"new-req"}
|
||||
assert "old" in self.worker._remote_agents
|
||||
assert self.worker._engine_by_address == {("localhost", 1234): "old"}
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"states,failed",
|
||||
[
|
||||
(["PROC", "DONE"], False),
|
||||
(["ERR", "PROC", "DONE"], True),
|
||||
(["ERR", "PROC", "ERR"], True),
|
||||
],
|
||||
ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"],
|
||||
)
|
||||
def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed):
|
||||
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
|
||||
self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1]
|
||||
self.transport.check_xfer_state.side_effect = states
|
||||
self._connect()
|
||||
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == result.failed_recving == set()
|
||||
assert self.worker.get_block_ids_with_load_errors() == set()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == {"old-req"}
|
||||
assert result.failed_recving == ({"old-req"} if failed else set())
|
||||
assert self.worker.get_block_ids_with_load_errors() == (
|
||||
{1} if failed else set()
|
||||
)
|
||||
assert self.transport.release_xfer_handle.call_count == (2 if failed else 1)
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
def test_waits_for_queued_notification(self):
|
||||
metadata = self._request("old", blocks=())
|
||||
self.worker._recving_metadata.update(metadata.reqs_to_recv)
|
||||
self.worker._background_nixl_handshake(
|
||||
"old-req", "old", metadata.reqs_to_recv["old-req"]
|
||||
)
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
self.worker.start_load_kv(NixlConnectorMetadata())
|
||||
self.transport.send_notif.assert_called_once_with(
|
||||
"old", notif_msg=b"old-prefill:1"
|
||||
)
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == {"old-req"}
|
||||
assert result.failed_recving == set()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
def test_waits_for_other_handshakes(self):
|
||||
self.worker._ensure_handshake("other", "other-host", 1234, 1)
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
self._connect("other", host="other-host")
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
def test_rehandshakes_queued_request_for_released_engine(self):
|
||||
# A request queued after its engine's handshake, but drained only after
|
||||
# that engine was replaced and released, must not read from it.
|
||||
metadata = self._request("old", blocks=(), awaiting_kvs=False)
|
||||
self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"]))
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
self.worker.start_load_kv(NixlConnectorMetadata())
|
||||
assert "old" in self.worker._handshake_futures
|
||||
self.transport.send_notif.assert_not_called()
|
||||
|
||||
|
||||
def test_transfer_topology_unregister():
|
||||
"""TransferTopology.unregister_remote_engine removes the engine."""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import gc
|
||||
import queue
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -54,6 +55,9 @@ def region_pull_worker():
|
||||
worker.enable_permute_local_kv = False
|
||||
worker.enable_heterogeneous_attn_post_process = False
|
||||
worker._engine_last_active = {}
|
||||
worker._handshake_lock = threading.RLock()
|
||||
worker._handshake_futures = {}
|
||||
worker._transfer_layer_group_ids = ()
|
||||
worker._bidirectional_kv_xfer_enabled = False
|
||||
worker._recving_transfers = {}
|
||||
worker.use_mla, worker._has_mamba = True, False
|
||||
@@ -94,6 +98,7 @@ def region_pull_worker():
|
||||
worker.dst_num_blocks = {"P": 100, "D": 100}
|
||||
worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]}
|
||||
worker._remote_agents = {"P": {(0, 0): "P-rank0"}}
|
||||
worker._engine_by_address = {("localhost", 1): "P"}
|
||||
worker._read_blocks_mixed = MagicMock()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
return worker
|
||||
|
||||
@@ -395,6 +395,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w._group_spec_types = (FullAttentionSpec,)
|
||||
w._engine_ttl = 0.0
|
||||
w._engine_last_active = {}
|
||||
w._engine_by_address = {}
|
||||
|
||||
# Track _do_start_push_kv invocations.
|
||||
calls: list[tuple[str, Any, dict[str, Any]]] = []
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for SimpleCPUOffloadConnector metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
|
||||
from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import (
|
||||
_make_connector,
|
||||
)
|
||||
from tests.v1.simple_kv_offload.test_scheduler import (
|
||||
make_request,
|
||||
make_scheduler,
|
||||
make_scheduler_output,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.simple_kv_offload.manager import (
|
||||
BoundaryStoreStats,
|
||||
LoadRequestState,
|
||||
TransferMeta,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
INFO_LABELS,
|
||||
OUTCOME_TO_FIELD,
|
||||
MetricName,
|
||||
SimpleCPUOffloadStats,
|
||||
_MetricType,
|
||||
_StatsKey,
|
||||
)
|
||||
|
||||
SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES
|
||||
LOAD_BLOCKS = MetricName.LOAD_BLOCKS
|
||||
USED_BLOCKS = MetricName.USED_BLOCKS
|
||||
PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS
|
||||
INFO = MetricName.INFO
|
||||
|
||||
|
||||
class _FakeMetric:
|
||||
def __init__(self, **kwargs: Any):
|
||||
self.kwargs = kwargs
|
||||
self.children: list[_FakeMetric] = []
|
||||
self.observed: list[int | float] = []
|
||||
self.increments: list[int | float] = []
|
||||
self.set_values: list[int | float] = []
|
||||
self.labelvalues: tuple[object, ...] = ()
|
||||
|
||||
def labels(self, *labelvalues: object) -> _FakeMetric:
|
||||
child = _FakeMetric(**self.kwargs)
|
||||
child.labelvalues = labelvalues
|
||||
self.children.append(child)
|
||||
return child
|
||||
|
||||
def observe(self, value: int | float) -> None:
|
||||
self.observed.append(value)
|
||||
|
||||
def inc(self, value: int | float) -> None:
|
||||
self.increments.append(value)
|
||||
|
||||
def set(self, value: int | float) -> None:
|
||||
self.set_values.append(value)
|
||||
|
||||
|
||||
def _fake_metric_types() -> dict[type, type]:
|
||||
return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric}
|
||||
|
||||
|
||||
def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any:
|
||||
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
|
||||
|
||||
return SimpleCPUOffloadPromMetrics(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=None),
|
||||
metric_types=_fake_metric_types(),
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues=per_engine or {0: ["model", "0"]},
|
||||
)
|
||||
|
||||
|
||||
def _stats_payload(types: dict, data: dict) -> dict[str, Any]:
|
||||
return {_StatsKey.TYPES: types, _StatsKey.DATA: data}
|
||||
|
||||
|
||||
def _values(stats: SimpleCPUOffloadStats, name: str) -> dict:
|
||||
return stats.data[_StatsKey.DATA][name]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stats class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_kv_connector_stats_none_and_empty() -> None:
|
||||
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None)
|
||||
assert isinstance(stats, SimpleCPUOffloadStats)
|
||||
assert stats.is_empty()
|
||||
|
||||
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={})
|
||||
assert isinstance(stats, SimpleCPUOffloadStats)
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_stats_round_trip_through_serialized_dict() -> None:
|
||||
stats = SimpleCPUOffloadStats()
|
||||
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats.increase_counter(LOAD_BLOCKS, 4)
|
||||
stats.set_gauge(USED_BLOCKS, 7)
|
||||
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
|
||||
|
||||
rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict())
|
||||
assert isinstance(rebuilt, SimpleCPUOffloadStats)
|
||||
assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3}
|
||||
assert _values(rebuilt, LOAD_BLOCKS) == {(): 4}
|
||||
assert _values(rebuilt, USED_BLOCKS) == {(): 7}
|
||||
assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1}
|
||||
|
||||
|
||||
def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None:
|
||||
stats1 = SimpleCPUOffloadStats()
|
||||
stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",))
|
||||
stats1.set_gauge(USED_BLOCKS, 7)
|
||||
|
||||
stats2 = SimpleCPUOffloadStats()
|
||||
stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",))
|
||||
stats2.set_gauge(USED_BLOCKS, 2)
|
||||
|
||||
result = stats1.aggregate(stats2)
|
||||
assert result is stats1
|
||||
outcomes = _values(stats1, SAVE_OUTCOMES)
|
||||
assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1}
|
||||
assert _values(stats1, USED_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_stats_aggregate_merges_types_from_other() -> None:
|
||||
stats1 = SimpleCPUOffloadStats()
|
||||
stats1.set_gauge(USED_BLOCKS, 1)
|
||||
|
||||
stats2 = SimpleCPUOffloadStats()
|
||||
stats2.increase_counter(LOAD_BLOCKS, 2)
|
||||
|
||||
stats1.aggregate(stats2)
|
||||
assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER
|
||||
assert _values(stats1, LOAD_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_stats_reduce_and_reset() -> None:
|
||||
stats = SimpleCPUOffloadStats()
|
||||
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats.set_gauge(PENDING_STORE_BLOCKS, 4)
|
||||
stats.increase_counter(LOAD_BLOCKS, 9)
|
||||
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
|
||||
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3
|
||||
assert reduced[PENDING_STORE_BLOCKS] == 4
|
||||
assert reduced[LOAD_BLOCKS] == 9
|
||||
assert not any(key.startswith(INFO) for key in reduced)
|
||||
|
||||
stats.reset()
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_outcome_labels_match_boundary_store_stats_fields() -> None:
|
||||
"""Every outcome label maps 1:1 to a BoundaryStoreStats counter field."""
|
||||
boundary_fields = {
|
||||
f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published"
|
||||
}
|
||||
assert set(OUTCOME_TO_FIELD.values()) == boundary_fields
|
||||
assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prom metrics class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prom_metrics_registers_tier1_metrics() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
defs = prom._defs
|
||||
assert set(defs) == {
|
||||
SAVE_OUTCOMES,
|
||||
LOAD_BLOCKS,
|
||||
USED_BLOCKS,
|
||||
PENDING_STORE_BLOCKS,
|
||||
INFO,
|
||||
}
|
||||
# The prom class registers literal names (required by the docs
|
||||
# generator); they must stay in sync with the emission-side constants.
|
||||
assert {m.kwargs["name"] for m in defs.values()} == {
|
||||
SAVE_OUTCOMES,
|
||||
LOAD_BLOCKS,
|
||||
USED_BLOCKS,
|
||||
PENDING_STORE_BLOCKS,
|
||||
INFO,
|
||||
}
|
||||
for metric in defs.values():
|
||||
assert metric.kwargs["documentation"]
|
||||
assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"]
|
||||
|
||||
assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [
|
||||
"model_name",
|
||||
"engine",
|
||||
"outcome",
|
||||
]
|
||||
assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
|
||||
assert defs[INFO].kwargs["labelnames"] == [
|
||||
"model_name",
|
||||
"engine",
|
||||
*INFO_LABELS,
|
||||
]
|
||||
assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
|
||||
|
||||
# mostrecent: multiprocess scrapes expose one series per label set, not
|
||||
# one pid-tagged series per writer process.
|
||||
for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO):
|
||||
assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent"
|
||||
|
||||
|
||||
def test_prom_metrics_observe_routes_each_series() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={
|
||||
SAVE_OUTCOMES: _MetricType.COUNTER,
|
||||
LOAD_BLOCKS: _MetricType.COUNTER,
|
||||
USED_BLOCKS: _MetricType.GAUGE,
|
||||
PENDING_STORE_BLOCKS: _MetricType.GAUGE,
|
||||
INFO: _MetricType.GAUGE,
|
||||
},
|
||||
data={
|
||||
SAVE_OUTCOMES: {("stored",): 3},
|
||||
LOAD_BLOCKS: {(): 4},
|
||||
USED_BLOCKS: {(): 5},
|
||||
PENDING_STORE_BLOCKS: {(): 1},
|
||||
INFO: {("cpu", "false", "false", "64"): 1},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3]
|
||||
assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4]
|
||||
assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5]
|
||||
assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1]
|
||||
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1]
|
||||
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == (
|
||||
"model",
|
||||
"0",
|
||||
"stored",
|
||||
)
|
||||
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == (
|
||||
"model",
|
||||
"0",
|
||||
"cpu",
|
||||
"false",
|
||||
"false",
|
||||
"64",
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_reuses_bound_children() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
payload = _stats_payload(
|
||||
types={SAVE_OUTCOMES: _MetricType.COUNTER},
|
||||
data={SAVE_OUTCOMES: {("stored",): 1}},
|
||||
)
|
||||
prom.observe(payload)
|
||||
prom.observe(payload)
|
||||
|
||||
child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))]
|
||||
assert child.increments == [1, 1]
|
||||
assert len(prom._defs[SAVE_OUTCOMES].children) == 1
|
||||
|
||||
|
||||
def test_prom_metrics_rejects_unknown_metric() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
with pytest.raises(AssertionError, match="Unknown"):
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={"vllm:not_a_metric": _MetricType.COUNTER},
|
||||
data={"vllm:not_a_metric": {(): 1}},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_rejects_wrong_label_count() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
with pytest.raises(AssertionError, match="labels"):
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={SAVE_OUTCOMES: _MetricType.COUNTER},
|
||||
data={SAVE_OUTCOMES: {("stored", "extra"): 1}},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_routes_per_engine() -> None:
|
||||
prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]})
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}}
|
||||
),
|
||||
engine_idx=1,
|
||||
)
|
||||
assert (0, USED_BLOCKS, ()) not in prom._metrics
|
||||
assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3]
|
||||
|
||||
|
||||
def test_connector_build_prom_metrics() -> None:
|
||||
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
|
||||
|
||||
prom = SimpleCPUOffloadConnector.build_prom_metrics(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=None),
|
||||
metric_types=_fake_metric_types(),
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"]},
|
||||
)
|
||||
assert isinstance(prom, SimpleCPUOffloadPromMetrics)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Manager: interval stats
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None:
|
||||
fixture = make_scheduler()
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched.boundary_store_stats.stored += 3
|
||||
sched.boundary_store_stats.dropped_cpu_full += 2
|
||||
sched.boundary_store_stats.skipped_in_flight += 1
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, SAVE_OUTCOMES) == {
|
||||
("stored",): 3,
|
||||
("dropped_cpu_full",): 2,
|
||||
("skipped_in_flight",): 1,
|
||||
}
|
||||
|
||||
# Deltas are per-interval: a second drain reports no new outcomes.
|
||||
stats2 = sched.get_stats()
|
||||
assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA]
|
||||
|
||||
|
||||
def test_manager_reset_preserves_undrained_outcome_deltas() -> None:
|
||||
fixture = make_scheduler()
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched.boundary_store_stats.stored += 3
|
||||
sched.reset()
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3}
|
||||
|
||||
|
||||
def test_manager_get_stats_gauges_and_info_labels() -> None:
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched._store_event_to_blocks[0] = TransferMeta([10], [11])
|
||||
sched._pending_finished_stores.append(TransferMeta([12], [13]))
|
||||
sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15])
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3}
|
||||
used = _values(stats, USED_BLOCKS)[()]
|
||||
assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks()
|
||||
|
||||
info = _values(stats, INFO)
|
||||
assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1}
|
||||
|
||||
# Gauges are re-reported every drain.
|
||||
stats2 = sched.get_stats()
|
||||
assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3}
|
||||
|
||||
|
||||
def test_manager_info_labels_reflect_disk_mode() -> None:
|
||||
connector = _make_connector(
|
||||
extra_config={
|
||||
"kv_offload_backend": "disk",
|
||||
"disk_path": "/tmp/fake-disk-metrics",
|
||||
"disk_capacity_bytes": 1024**3,
|
||||
"use_page_cache": True,
|
||||
}
|
||||
)
|
||||
sched = connector.scheduler_manager
|
||||
assert sched is not None
|
||||
assert sched._info_labelvalues[0] == "disk"
|
||||
assert sched._info_labelvalues[1] == "true"
|
||||
stats = sched.get_stats()
|
||||
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
|
||||
|
||||
|
||||
def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None:
|
||||
"""CPU backend ignores use_page_cache; the info gauge must not claim it."""
|
||||
connector = _make_connector(extra_config={"use_page_cache": True})
|
||||
sched = connector.scheduler_manager
|
||||
assert sched is not None
|
||||
assert sched._info_labelvalues[:2] == ("cpu", "false")
|
||||
stats = sched.get_stats()
|
||||
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
|
||||
|
||||
|
||||
def test_manager_counts_load_blocks_completed() -> None:
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
request = make_request(num_blocks=2)
|
||||
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
|
||||
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
|
||||
sched._reqs_to_load[request.request_id] = LoadRequestState(
|
||||
request=request,
|
||||
transfer_meta=TransferMeta(
|
||||
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
|
||||
),
|
||||
)
|
||||
|
||||
sched.build_connector_meta(make_scheduler_output({}))
|
||||
stats = sched.get_stats()
|
||||
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
|
||||
|
||||
sched.update_connector_output(
|
||||
KVConnectorOutput(finished_recving={request.request_id})
|
||||
)
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, LOAD_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_manager_abandoned_load_counts_as_completed_after_reset() -> None:
|
||||
"""Loads abandoned by reset() that still finish on the worker are
|
||||
counted as loaded blocks.
|
||||
"""
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
request = make_request(num_blocks=2)
|
||||
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
|
||||
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
|
||||
sched._reqs_to_load[request.request_id] = LoadRequestState(
|
||||
request=request,
|
||||
transfer_meta=TransferMeta(
|
||||
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
|
||||
),
|
||||
)
|
||||
|
||||
sched.build_connector_meta(make_scheduler_output({}))
|
||||
stats = sched.get_stats()
|
||||
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
|
||||
|
||||
assert sched.reset() is False
|
||||
assert set(sched._abandoned_reqs_to_load) == {request.request_id}
|
||||
|
||||
sched.update_connector_output(
|
||||
KVConnectorOutput(finished_recving={request.request_id})
|
||||
)
|
||||
assert sched._abandoned_reqs_to_load == {}
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, LOAD_BLOCKS) == {(): 2}
|
||||
@@ -2316,8 +2316,8 @@ def _make_hybrid_attention_mamba_scheduler(
|
||||
def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
|
||||
"""Replicated groups must not be scaled by the DCP world size.
|
||||
|
||||
``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP
|
||||
size and every other spec 1. Reconstructing a group's geometry as
|
||||
The spec's ``dcp_sharded`` flag controls its effective block size.
|
||||
Reconstructing a group's geometry as
|
||||
``spec.block_size * cp_world_size`` over-scales the replicated groups, so
|
||||
the eager store scan believes far fewer of their blocks are ready and
|
||||
silently offloads only a fraction of them.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -9,7 +10,7 @@ import torch
|
||||
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
|
||||
prepare_dflash_inputs,
|
||||
DFlashSpeculator,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -24,6 +25,7 @@ def _run_prepare(
|
||||
cp_rank: int = 0,
|
||||
cp_size: int = 1,
|
||||
cp_interleave: int = 1,
|
||||
draft_dcp_size: int | None = None,
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
max_num_reqs = 4
|
||||
@@ -40,7 +42,9 @@ def _run_prepare(
|
||||
)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
num_tokens=4,
|
||||
num_scheduled_tokens=np.array([4], dtype=np.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([target_positions[-1] + 1]),
|
||||
positions=torch.tensor(target_positions, dtype=torch.int64, device=device),
|
||||
query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device),
|
||||
idx_mapping=torch.tensor([2], dtype=torch.int32, device=device),
|
||||
@@ -74,36 +78,65 @@ def _run_prepare(
|
||||
next_prefill_tokens = torch.zeros_like(last_sampled)
|
||||
block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device)
|
||||
|
||||
prepare_dflash_inputs(
|
||||
input_buffers,
|
||||
query_slot_mapping,
|
||||
context_positions,
|
||||
context_slot_mapping,
|
||||
sample_indices,
|
||||
sample_pos,
|
||||
sample_idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
input_batch,
|
||||
torch.tensor([1], dtype=torch.int32, device=device),
|
||||
torch.tensor([2], dtype=torch.int32, device=device),
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
input_temperature,
|
||||
input_seeds,
|
||||
block_table,
|
||||
4,
|
||||
cp_rank,
|
||||
cp_size,
|
||||
cp_interleave,
|
||||
123,
|
||||
num_speculative_steps,
|
||||
num_speculative_steps,
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
128,
|
||||
class InputsPrepared(Exception):
|
||||
pass
|
||||
|
||||
draft = SimpleNamespace(
|
||||
input_buffers=input_buffers,
|
||||
context_positions=context_positions,
|
||||
_context_slot_mappings=context_slot_mapping.unsqueeze(0),
|
||||
sample_indices=sample_indices,
|
||||
sample_pos=sample_pos,
|
||||
sample_idx_mapping=sample_idx_mapping,
|
||||
temperature=temperature,
|
||||
seeds=seeds,
|
||||
hidden_states=torch.zeros(4, 1, device=device),
|
||||
prepare_context_anchor=lambda *args: None,
|
||||
query_cudagraph_manager=None,
|
||||
dp_size=1,
|
||||
dp_rank=0,
|
||||
pcp_manager=None,
|
||||
draft_kv_cache_group_id=0,
|
||||
draft_kv_cache_group_ids=[0],
|
||||
_layer_group_idx=None,
|
||||
block_tables=SimpleNamespace(
|
||||
slot_mappings=query_slot_mapping.unsqueeze(0),
|
||||
input_block_tables=[block_table],
|
||||
kernel_block_sizes=[4],
|
||||
cp_rank=cp_rank,
|
||||
cp_size=cp_size,
|
||||
cp_interleave=cp_interleave,
|
||||
),
|
||||
dcp_size=cp_size if draft_dcp_size is None else draft_dcp_size,
|
||||
parallel_drafting_token_id=123,
|
||||
num_query_per_req=num_speculative_steps,
|
||||
num_speculative_steps=num_speculative_steps,
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_num_tokens=max_num_tokens,
|
||||
max_model_len=128,
|
||||
sample_from_anchor=True,
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"vllm.v1.worker.gpu.spec_decode.dflash.speculator.dispatch_cg_and_sync_dp",
|
||||
side_effect=InputsPrepared,
|
||||
),
|
||||
pytest.raises(InputsPrepared),
|
||||
):
|
||||
DFlashSpeculator.propose(
|
||||
draft,
|
||||
input_batch,
|
||||
{},
|
||||
{},
|
||||
draft.hidden_states,
|
||||
None,
|
||||
torch.tensor([1], dtype=torch.int32, device=device),
|
||||
torch.tensor([2], dtype=torch.int32, device=device),
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
input_temperature,
|
||||
input_seeds,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
return SimpleNamespace(
|
||||
input_buffers=input_buffers,
|
||||
@@ -118,12 +151,17 @@ def _run_prepare(
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
|
||||
@pytest.mark.parametrize("cp_rank,cp_size", [(0, 1), (0, 2), (1, 2)])
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size):
|
||||
# Positions 10/11 use physical block 7. Rejected positions 12/13 would use
|
||||
# block 8, but must be PAD context rather than contaminating draft KV.
|
||||
out = _run_prepare(
|
||||
target_positions=[10, 11, 12, 13],
|
||||
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
|
||||
cp_rank=cp_rank,
|
||||
cp_size=cp_size,
|
||||
cp_interleave=2,
|
||||
draft_dcp_size=1,
|
||||
)
|
||||
|
||||
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
|
||||
|
||||
@@ -17,6 +17,7 @@ from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
@@ -53,6 +54,7 @@ def _make_fake_speculator(
|
||||
arange_np=np.arange(max_num_reqs + 1, dtype=np.int32),
|
||||
draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool),
|
||||
block_tables=fake_block_tables,
|
||||
dcp_size=1,
|
||||
input_buffers=fake_input_buffers,
|
||||
attn_groups=[],
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
|
||||
@@ -152,14 +154,17 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len():
|
||||
assert torch.equal(bound, torch.tensor([1024, 503], dtype=torch.int32))
|
||||
|
||||
|
||||
def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
|
||||
@pytest.mark.parametrize("draft_dcp_size", [1, 2])
|
||||
def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(draft_dcp_size):
|
||||
fake = _make_fake_speculator()
|
||||
fake.dcp_size = draft_dcp_size
|
||||
fake.block_tables.cp_size = 2
|
||||
fake.block_tables.cp_rank = 1
|
||||
fake.block_tables.cp_interleave = 4
|
||||
fake.input_buffers.seq_lens[:3] = torch.tensor([5, 9, 16])
|
||||
|
||||
def fake_prepare(out, seq_lens, num_reqs, dcp_size, dcp_rank, cp_interleave):
|
||||
assert draft_dcp_size > 1
|
||||
assert seq_lens is fake.input_buffers.seq_lens
|
||||
assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (3, 2, 1, 4)
|
||||
out[:num_reqs].copy_(torch.tensor([1, 4, 8], dtype=torch.int32))
|
||||
@@ -176,6 +181,9 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
|
||||
)
|
||||
|
||||
local = captured["dcp_local_seq_lens"]
|
||||
if draft_dcp_size == 1:
|
||||
assert local is None
|
||||
return
|
||||
assert isinstance(local, torch.Tensor)
|
||||
assert local.data_ptr() == fake.input_buffers.dcp_local_seq_lens.data_ptr()
|
||||
assert local.tolist() == [1, 4, 8, 0]
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -7,6 +9,38 @@ from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
|
||||
from vllm.v1.worker.cp_utils import should_skip_dcp_context_attention
|
||||
|
||||
|
||||
def test_cp_checks_allow_a_replicated_draft_backend_without_dcp(monkeypatch):
|
||||
from vllm.v1.worker import cp_utils
|
||||
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=1,
|
||||
decode_context_parallel_size=4,
|
||||
cp_kv_cache_interleave_size=16,
|
||||
),
|
||||
speculative_config=object(),
|
||||
)
|
||||
target = SimpleNamespace(
|
||||
impl=SimpleNamespace(
|
||||
dcp_world_size=4,
|
||||
supports_mtp_with_cp_non_trivial_interleave_size=True,
|
||||
need_to_return_lse_for_decode=True,
|
||||
)
|
||||
)
|
||||
draft = SimpleNamespace(
|
||||
impl=SimpleNamespace(
|
||||
dcp_world_size=1,
|
||||
supports_mtp_with_cp_non_trivial_interleave_size=False,
|
||||
need_to_return_lse_for_decode=False,
|
||||
)
|
||||
)
|
||||
layers = {"target": target, "draft": draft}
|
||||
monkeypatch.setattr(cp_utils, "get_layers_from_vllm_config", lambda *args: layers)
|
||||
cp_utils.check_attention_cp_compatibility(config, target_layer_names={"target"})
|
||||
with pytest.raises(AssertionError, match="MTP with cp_kv_cache_interleave_size"):
|
||||
cp_utils.check_attention_cp_compatibility(config)
|
||||
|
||||
|
||||
def test_skip_gate_only_for_zero_context():
|
||||
assert should_skip_dcp_context_attention(torch.zeros(3, dtype=torch.int32))
|
||||
assert not should_skip_dcp_context_attention(
|
||||
|
||||
@@ -2,8 +2,17 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
from vllm.v1.worker.gpu import model_runner
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_config
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -49,3 +58,86 @@ def test_dspark_parallel_config_disables_eplb_atomically():
|
||||
assert draft_config.eplb_config.num_redundant_experts == 0
|
||||
assert not draft_config.enable_elastic_ep
|
||||
assert draft_config.eplb_config is not target_config.eplb_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size", [1, 4])
|
||||
@pytest.mark.parametrize("dcp_size", [1, 4])
|
||||
@pytest.mark.parametrize("use_mla", [False, True])
|
||||
def test_draft_context_parallelism_without_changing_target(
|
||||
monkeypatch, pcp_size, dcp_size, use_mla
|
||||
):
|
||||
target_parallel = ParallelConfig(
|
||||
tensor_parallel_size=4,
|
||||
prefill_context_parallel_size=pcp_size,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=16,
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
target_config = SimpleNamespace(
|
||||
parallel_config=target_parallel,
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(use_mla=use_mla)
|
||||
),
|
||||
)
|
||||
|
||||
class CapturedConfig(Exception):
|
||||
pass
|
||||
|
||||
def capture_init(self, config, device):
|
||||
raise CapturedConfig(config)
|
||||
|
||||
monkeypatch.setattr(DraftModelSpeculator, "__init__", capture_init)
|
||||
with pytest.raises(CapturedConfig) as captured:
|
||||
DFlashSpeculator(target_config, device=None)
|
||||
draft_parallel = captured.value.args[0].parallel_config
|
||||
assert draft_parallel.tensor_parallel_size == 4
|
||||
assert draft_parallel.prefill_context_parallel_size == 1
|
||||
assert draft_parallel.decode_context_parallel_size == (dcp_size if use_mla else 1)
|
||||
assert draft_parallel.cp_kv_cache_interleave_size == 16
|
||||
assert target_parallel.prefill_context_parallel_size == pcp_size
|
||||
assert target_parallel.decode_context_parallel_size == dcp_size
|
||||
assert target_parallel.cp_kv_cache_interleave_size == 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured_dcp", [1, 4])
|
||||
def test_attention_uses_draft_dcp_setting_inside_target_process_group(
|
||||
monkeypatch, configured_dcp
|
||||
):
|
||||
import vllm.config as config_module
|
||||
from vllm.distributed import parallel_state
|
||||
from vllm.v1.attention.backend import AttentionImplBase
|
||||
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=configured_dcp)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
config_module, "get_current_vllm_config_or_none", lambda: config
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
parallel_state, "_DCP", SimpleNamespace(world_size=4, rank_in_group=2)
|
||||
)
|
||||
impl = AttentionImplBase()
|
||||
assert impl.dcp_world_size == configured_dcp
|
||||
assert impl.dcp_rank == (0 if configured_dcp == 1 else 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("draft_dcp_size", [1, 4])
|
||||
def test_runner_marks_only_replicated_draft_caches(monkeypatch, draft_dcp_size):
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.bfloat16
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_runner, "get_kv_cache_spec", lambda _: {"target": spec, "draft": spec}
|
||||
)
|
||||
runner = object.__new__(model_runner.GPUModelRunner)
|
||||
runner.vllm_config = None
|
||||
runner.dcp_size = 4
|
||||
runner.speculator = object.__new__(DFlashSpeculator)
|
||||
runner.speculator.dcp_size = draft_dcp_size
|
||||
runner.speculator.draft_attn_layer_names = {"draft"}
|
||||
|
||||
specs = runner.get_kv_cache_spec()
|
||||
|
||||
assert specs["target"].dcp_sharded
|
||||
assert specs["draft"].dcp_sharded == (draft_dcp_size == 4)
|
||||
|
||||
@@ -88,6 +88,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
|
||||
query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32),
|
||||
)
|
||||
speculator.dcp_size = 1
|
||||
speculator.block_tables = SimpleNamespace(
|
||||
cp_size=1,
|
||||
input_block_tables=[torch.zeros(4, 1, dtype=torch.int32)],
|
||||
@@ -183,6 +184,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected)
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=1,
|
||||
data_parallel_rank=0,
|
||||
decode_context_parallel_size=1,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -171,22 +171,23 @@ def test_block_tables_skip_custom_slot_mapping_groups():
|
||||
|
||||
@pytest.mark.parametrize("cp_rank", range(4))
|
||||
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
"""DCP interleave is expressed in logical-block token coordinates."""
|
||||
"""Only sharded groups use DCP interleave in logical-block coordinates."""
|
||||
device = torch.device("cuda")
|
||||
block_tables = BlockTables(
|
||||
block_sizes=[128],
|
||||
block_sizes=[128, 128],
|
||||
max_num_reqs=1,
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_blocks_per_group=[2],
|
||||
max_num_blocks_per_group=[2, 8],
|
||||
device=device,
|
||||
kernel_block_sizes=[64],
|
||||
kernel_block_sizes=[64, 64],
|
||||
dcp_sharded=[True, False],
|
||||
cp_size=4,
|
||||
cp_rank=cp_rank,
|
||||
cp_interleave=128,
|
||||
)
|
||||
block_tables.append_block_ids(
|
||||
req_index=0,
|
||||
new_block_ids=([5, 9],),
|
||||
new_block_ids=([5, 9], list(range(10, 18))),
|
||||
overwrite=True,
|
||||
)
|
||||
block_tables.apply_staged_writes()
|
||||
@@ -199,7 +200,7 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_tokens_padded=1024,
|
||||
)[0]
|
||||
)
|
||||
|
||||
expected = torch.full((1024,), -1, dtype=torch.int64, device=device)
|
||||
first_start = cp_rank * 128
|
||||
@@ -210,7 +211,8 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
expected[second_start : second_start + 128] = torch.arange(
|
||||
9 * 128, 10 * 128, dtype=torch.int64, device=device
|
||||
)
|
||||
assert torch.equal(actual, expected)
|
||||
assert torch.equal(actual[0], expected)
|
||||
assert torch.equal(actual[1], positions + 10 * 128)
|
||||
|
||||
|
||||
def test_v1_block_table_move_row_clears_vacated_row():
|
||||
|
||||
@@ -372,6 +372,62 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
_compare_objs(input_batch, ref_input_batch)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_condense_clears_stale_allowed_token_ids_mask(device: str):
|
||||
"""condense() must clear the mask row a constrained request is moved out
|
||||
of. Otherwise a later request that reuses that row without
|
||||
allowed_token_ids inherits the stale whitelist.
|
||||
See https://github.com/vllm-project/vllm/issues/43894.
|
||||
"""
|
||||
batch_size = 4
|
||||
allowed_token_id = 13
|
||||
input_batch = InputBatch(
|
||||
max_num_reqs=batch_size,
|
||||
max_model_len=1024,
|
||||
max_num_batched_tokens=1024,
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
def _make_req(suffix: int, allowed_token_ids=None) -> CachedRequestState:
|
||||
return CachedRequestState(
|
||||
req_id=f"req_id_{suffix}",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(allowed_token_ids=allowed_token_ids),
|
||||
pooling_params=None,
|
||||
mm_features=[],
|
||||
block_ids=([],),
|
||||
generator=None,
|
||||
num_computed_tokens=0,
|
||||
output_token_ids=[],
|
||||
)
|
||||
|
||||
# Only req_id_2 is constrained, and it lands on the highest row.
|
||||
assert input_batch.add_request(_make_req(0)) == 0
|
||||
assert input_batch.add_request(_make_req(1)) == 1
|
||||
assert input_batch.add_request(_make_req(2, [allowed_token_id])) == 2
|
||||
|
||||
mask = input_batch.allowed_token_ids_mask_cpu_tensor
|
||||
assert mask is not None
|
||||
# The constrained row masks every token except the single allowed id.
|
||||
assert not mask[2][allowed_token_id].item()
|
||||
assert int(mask[2].sum().item()) == VOCAB_SIZE - 1
|
||||
|
||||
# Free row 0, then condense: req_id_2 slides from row 2 down into row 0.
|
||||
input_batch.remove_request("req_id_0")
|
||||
input_batch.condense()
|
||||
assert input_batch.req_id_to_index["req_id_2"] == 0
|
||||
assert int(mask[2].sum().item()) == 0
|
||||
|
||||
# A new unrestricted request reuses row 2 and must stay unconstrained.
|
||||
assert input_batch.add_request(_make_req(3)) == 2
|
||||
assert "req_id_3" not in input_batch.has_allowed_token_ids
|
||||
assert int(mask[2].sum().item()) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_swap_states_preserves_allowed_token_ids_mask(device: str):
|
||||
"""swap_states() must exchange the two allowed_token_ids_mask rows.
|
||||
|
||||
@@ -75,6 +75,24 @@ def test_validate_config_rejects_full_graph_for_prefills():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("draft_uses_mla", [False, True])
|
||||
@pytest.mark.parametrize("dcp_size", [1, 2, 4])
|
||||
def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size):
|
||||
config = _make_config(CUDAGraphMode.NONE)
|
||||
config.parallel_config.prefill_context_parallel_size = 4
|
||||
config.parallel_config.decode_context_parallel_size = dcp_size
|
||||
config.model_config.hf_text_config.index_topk = 1
|
||||
config.speculative_config = SimpleNamespace(
|
||||
use_dspark=lambda: True,
|
||||
draft_model_config=SimpleNamespace(use_mla=draft_uses_mla),
|
||||
)
|
||||
if draft_uses_mla and dcp_size == 2:
|
||||
with pytest.raises(NotImplementedError, match="requires DCP=1 or DCP=PCP"):
|
||||
PCPManager.validate_config(config, supports_mm_inputs=False)
|
||||
return
|
||||
PCPManager.validate_config(config, supports_mm_inputs=False)
|
||||
|
||||
|
||||
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
|
||||
+51
-23
@@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail(
|
||||
_OPS_REGISTERED = False
|
||||
|
||||
|
||||
_AITER_SITUV2_ACT_ENV = {
|
||||
"a8w4": "AITER_SITUV2_A8W4",
|
||||
"a4w4": "AITER_SITUV2_A4W4",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_situv2_activation() -> str:
|
||||
"""VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4.
|
||||
|
||||
auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4.
|
||||
"""
|
||||
value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower()
|
||||
if value in ("auto", "1"):
|
||||
return "a4w4"
|
||||
if value == "0":
|
||||
return "a16w4"
|
||||
return value
|
||||
|
||||
|
||||
def _sync_aiter_situv2_moe_env() -> None:
|
||||
"""Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env.
|
||||
"""Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2.
|
||||
|
||||
AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 /
|
||||
AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first).
|
||||
When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4
|
||||
(afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override.
|
||||
|
||||
Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4
|
||||
dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs
|
||||
afp4 FlyDSL but falls back to heuristic configs, which is not the
|
||||
tuned a4w4 path this recipe is meant to use.
|
||||
AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4,
|
||||
so exactly one is set for a8w4/a4w4 and both are cleared for a16w4.
|
||||
"""
|
||||
import os
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2:
|
||||
os.environ["AITER_SITUV2_A4W4"] = "1"
|
||||
os.environ.pop("AITER_SITUV2_A8W4", None)
|
||||
else:
|
||||
os.environ.pop("AITER_SITUV2_A4W4", None)
|
||||
selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation())
|
||||
for name in _AITER_SITUV2_ACT_ENV.values():
|
||||
if name == selected:
|
||||
os.environ[name] = "1"
|
||||
else:
|
||||
os.environ.pop(name, None)
|
||||
|
||||
|
||||
class rocm_aiter_ops:
|
||||
@@ -1917,7 +1928,8 @@ class rocm_aiter_ops:
|
||||
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
|
||||
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4).
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation
|
||||
dtype (a16w4 | a8w4 | a4w4).
|
||||
VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM.
|
||||
|
||||
Note:
|
||||
@@ -1987,7 +1999,7 @@ class rocm_aiter_ops:
|
||||
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
|
||||
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
_MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
|
||||
_MOE_SITUV2 = _resolve_situv2_activation()
|
||||
# TODO: Consolidate under _LINEAR_ENABLED
|
||||
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
# Lazily probed: whether aiter.topk_softmax supports the
|
||||
@@ -2016,7 +2028,7 @@ class rocm_aiter_ops:
|
||||
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
|
||||
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
|
||||
cls._MOE_SITUV2 = _resolve_situv2_activation()
|
||||
_sync_aiter_situv2_moe_env()
|
||||
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
|
||||
@@ -2132,9 +2144,25 @@ class rocm_aiter_ops:
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_fused_moe_situv2_enabled(cls) -> bool:
|
||||
# _MOE_SITUV2 is a variant of aiter fused moe, so aiter
|
||||
# fused moe must be enabled as well.
|
||||
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2
|
||||
"""True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected."""
|
||||
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4"
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def get_fused_moe_situv2_activation(cls) -> str:
|
||||
"""SiTUv2 activation dtype name: a16w4, a8w4 or a4w4."""
|
||||
return cls._MOE_SITUV2
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool:
|
||||
"""Whether the SiTUv2 kernels expect gate/up-interleaved w13.
|
||||
|
||||
The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and
|
||||
a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode
|
||||
(experts/rocm_aiter_moe.py) must both use this.
|
||||
"""
|
||||
return cls._MOE_SITUV2 == "a8w4"
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
|
||||
@@ -1725,6 +1725,26 @@ class VllmConfig:
|
||||
)
|
||||
self.compilation_config.mode = CompilationMode.NONE
|
||||
|
||||
# TODO: This is a stopgap that turns sequence parallelism / async TP
|
||||
# off under batch invariance. Make the sequence-parallel reduce-scatter
|
||||
# path batch-invariant instead so they can stay enabled (#56370).
|
||||
# Sequence parallelism / async TP rewrite all_reduce + rms_norm into a
|
||||
# reduce-scatter whose reduction order depends on the batch, so they
|
||||
# are not batch-invariant. Decide this before the breakable-CUDA-graph
|
||||
# default below, which declines to auto-enable when they are on.
|
||||
pass_config = self.compilation_config.pass_config
|
||||
if envs.VLLM_BATCH_INVARIANT and (
|
||||
pass_config.enable_sp or pass_config.fuse_gemm_comms
|
||||
):
|
||||
logger.warning_once(
|
||||
"Disabling sequence parallelism and async TP "
|
||||
"(pass_config.enable_sp / fuse_gemm_comms) when "
|
||||
"VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path "
|
||||
"is not batch-invariant (see vllm-project/vllm#56370)."
|
||||
)
|
||||
pass_config.enable_sp = False
|
||||
pass_config.fuse_gemm_comms = False
|
||||
|
||||
breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph()
|
||||
|
||||
if not breakable_cudagraph_enabled and (
|
||||
|
||||
@@ -42,6 +42,7 @@ The class provides the following primitives:
|
||||
import enum
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
@@ -227,6 +228,13 @@ class KVConnectorBase_V1(ABC):
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a custom KV cache allocation context, if configured.
|
||||
|
||||
Returning None uses the engine's default memory pool.
|
||||
"""
|
||||
return None
|
||||
|
||||
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
|
||||
"""Set the connector metadata from the scheduler.
|
||||
|
||||
|
||||
@@ -59,7 +59,6 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
dcp_world_size_for_kv_cache_spec,
|
||||
resolve_dcp_kv_block_size,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -259,13 +258,7 @@ class DecodeBenchConnectorScheduler:
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
self.kv_cache_groups = kv_cache_config.kv_cache_groups
|
||||
self.group_block_sizes = tuple(
|
||||
resolve_dcp_kv_block_size(
|
||||
group.kv_cache_spec,
|
||||
dcp_world_size_for_kv_cache_spec(
|
||||
group.kv_cache_spec,
|
||||
dcp_world_size,
|
||||
),
|
||||
)
|
||||
resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_world_size)
|
||||
for group in self.kv_cache_groups
|
||||
)
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ enabling prefix caching via hash-based deduplication.
|
||||
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Sequence
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -322,6 +323,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Worker-side methods
|
||||
# ============================================================
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a context manager for the custom MemPool, or None.
|
||||
|
||||
Called by the Worker before ``initialize_kv_cache`` so that KV
|
||||
cache is allocated from the Mooncake-managed pool when
|
||||
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
|
||||
"""
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_mem_pool_context()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
@@ -19,13 +19,14 @@ import threading
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_dcp_group,
|
||||
get_pcp_group,
|
||||
@@ -1414,6 +1415,41 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
class MooncakeStoreWorker:
|
||||
"""Worker-side component for MooncakeStoreConnector."""
|
||||
|
||||
@staticmethod
|
||||
def _create_mem_pool(
|
||||
extra_config: dict[str, Any], model_config: ModelConfig
|
||||
) -> torch.cuda.MemPool | None:
|
||||
pool_type = str(extra_config.get("custom_mem_pool") or "").upper()
|
||||
if not pool_type:
|
||||
return None
|
||||
if pool_type not in ("NVLINK", "BAREX"):
|
||||
raise ValueError(
|
||||
f"Unsupported custom_mem_pool={pool_type!r}, "
|
||||
"expected 'NVLINK' or 'BAREX'"
|
||||
)
|
||||
if model_config.enable_sleep_mode or model_config.enable_cumem_allocator:
|
||||
raise ValueError(
|
||||
"custom_mem_pool is incompatible with enable_sleep_mode "
|
||||
"or enable_cumem_allocator; CuMemAllocator cannot manage "
|
||||
"allocations from the custom pool."
|
||||
)
|
||||
try:
|
||||
if pool_type == "NVLINK":
|
||||
from mooncake.allocator import NVLinkAllocator as allocator_cls
|
||||
else: # pool_type == "BAREX"
|
||||
from mooncake.allocator import BarexAllocator as allocator_cls
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
f"custom_mem_pool={pool_type!r} requires "
|
||||
"mooncake-transfer-engine>=0.3.8. Please upgrade Mooncake."
|
||||
) from e
|
||||
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
allocator = allocator_cls.get_allocator(device)
|
||||
mem_pool = torch.cuda.MemPool(allocator.allocator())
|
||||
logger.info("Using Mooncake custom memory pool: %s", pool_type)
|
||||
return mem_pool
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -1471,6 +1507,10 @@ class MooncakeStoreWorker:
|
||||
|
||||
# Initialize MooncakeDistributedStore with its own TransferEngine
|
||||
store_config = MooncakeStoreConfig.load_from_config()
|
||||
# Validate and create the custom memory pool before initializing the
|
||||
# store so failures cannot leak a partially initialized store handle.
|
||||
self._mem_pool = self._create_mem_pool(extra_config, model_config)
|
||||
|
||||
self.store = MooncakeDistributedStore()
|
||||
local_ip = get_ip()
|
||||
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
|
||||
@@ -1741,6 +1781,13 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
return token_dbs
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a context manager for the custom MemPool, or None if
|
||||
no custom pool is configured."""
|
||||
if self._mem_pool is None:
|
||||
return None
|
||||
return torch.cuda.use_mem_pool(self._mem_pool)
|
||||
|
||||
def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int:
|
||||
if self.dcp_size > 1:
|
||||
return 1
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
@@ -294,6 +295,26 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# ==============================
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Forward a custom KV cache memory pool from a child connector.
|
||||
|
||||
KV cache is allocated once and can only live in one pool, so at
|
||||
most one child may provide a context.
|
||||
"""
|
||||
found: list[tuple[str, AbstractContextManager]] = []
|
||||
for connector in self._connectors:
|
||||
if (ctx := connector.get_mem_pool_context()) is not None:
|
||||
found.append((type(connector).__name__, ctx))
|
||||
|
||||
if len(found) > 1:
|
||||
names = [name for name, _ in found]
|
||||
raise ValueError(
|
||||
f"Multiple connectors provide a KV cache memory pool {names}; "
|
||||
"KV cache is allocated once and can only live in one pool. "
|
||||
"Configure custom_mem_pool on at most one connector."
|
||||
)
|
||||
return found[0][1] if found else None
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
for c in self._connectors:
|
||||
c.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
@@ -839,6 +839,8 @@ class NixlBaseConnectorWorker:
|
||||
self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"engine_ttl", 3600.0
|
||||
)
|
||||
# Map of pull peer (host, port) -> engine_id currently serving it.
|
||||
self._engine_by_address: dict[tuple[str, int], EngineId] = {}
|
||||
|
||||
self.model_config = vllm_config.model_config
|
||||
|
||||
@@ -1300,6 +1302,7 @@ class NixlBaseConnectorWorker:
|
||||
self._remote_agents[eid] = remote_agents
|
||||
self._engine_clock_offset[eid] = clock_offset
|
||||
self._engine_last_active[eid] = time.perf_counter()
|
||||
self._track_remote_engine_replacement(eid, host, port)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="handshake_setup_failed",
|
||||
@@ -2953,6 +2956,7 @@ class NixlBaseConnectorWorker:
|
||||
# Handle timeout to avoid stranding blocks on remote.
|
||||
self._reap_expired_send_leases(done_sending)
|
||||
|
||||
self._cleanup_replaced_remote_engines()
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=done_sending,
|
||||
finished_recving=done_recving,
|
||||
@@ -3541,14 +3545,53 @@ class NixlBaseConnectorWorker:
|
||||
and meta.remote is not None
|
||||
}
|
||||
|
||||
def _track_remote_engine_replacement(
|
||||
self, engine_id: EngineId, host: str, port: int
|
||||
) -> None:
|
||||
"""Record the engine currently serving a pull peer address."""
|
||||
# TODO: Also handle push mode, which handshakes in both directions.
|
||||
if self._TRANSFER_MODE != "pull":
|
||||
return
|
||||
previous = self._engine_by_address.get((host, port))
|
||||
self._engine_by_address[(host, port)] = engine_id
|
||||
if previous is None or previous == engine_id:
|
||||
return
|
||||
logger.info(
|
||||
"Remote engine at %s:%d changed from %s to %s.",
|
||||
host,
|
||||
port,
|
||||
previous,
|
||||
engine_id,
|
||||
)
|
||||
|
||||
def _cleanup_replaced_remote_engines(self) -> None:
|
||||
"""Release replaced pull peers once requests and handshakes have drained."""
|
||||
if self._TRANSFER_MODE != "pull":
|
||||
return
|
||||
with self._handshake_lock:
|
||||
if self._handshake_futures:
|
||||
return
|
||||
replaced = self._remote_agents.keys() - self._engine_by_address.values()
|
||||
if not replaced:
|
||||
return
|
||||
# _recving_metadata is only accessed from this (main) thread.
|
||||
busy = {
|
||||
meta.remote.engine_id
|
||||
for meta in self._recving_metadata.values()
|
||||
if meta.remote is not None
|
||||
}
|
||||
for engine_id in replaced - busy:
|
||||
self._cleanup_remote_engine(engine_id, log_eviction=False)
|
||||
logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
|
||||
|
||||
def _cleanup_remote_engine(
|
||||
self, engine_id: EngineId, *, log_eviction: bool = True
|
||||
) -> None:
|
||||
"""Remove all state for a single remote engine.
|
||||
|
||||
Releases NIXL resources (dlist handles, remote agents) and clears
|
||||
all per-engine data structures. Used by both TTL eviction and
|
||||
shutdown.
|
||||
all per-engine data structures. Used by TTL eviction, replaced peer
|
||||
cleanup and shutdown.
|
||||
"""
|
||||
assert engine_id in self._remote_agents
|
||||
|
||||
@@ -3558,6 +3601,9 @@ class NixlBaseConnectorWorker:
|
||||
# Pop under the handshake lock; NIXL teardown stays outside it.
|
||||
with self._handshake_lock:
|
||||
agents = self._remote_agents.pop(engine_id)
|
||||
for address, eid in list(self._engine_by_address.items()):
|
||||
if eid == engine_id:
|
||||
del self._engine_by_address[address]
|
||||
for agent_name in agents.values():
|
||||
self.nixl_wrapper.remove_remote_agent(agent_name)
|
||||
|
||||
|
||||
@@ -84,7 +84,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
|
||||
# Start transfers for requests whose handshakes have now finished.
|
||||
while not self._ready_requests.empty():
|
||||
self._read_blocks_for_req(*self._ready_requests.get_nowait())
|
||||
req_id, meta = self._ready_requests.get_nowait()
|
||||
assert meta.remote is not None
|
||||
if meta.remote.engine_id not in self._remote_agents:
|
||||
# The engine was released after its handshake completed, so
|
||||
# handshake again. This fails if the engine is gone.
|
||||
self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
|
||||
continue
|
||||
self._read_blocks_for_req(req_id, meta)
|
||||
|
||||
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
|
||||
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
|
||||
|
||||
@@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
@@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import (
|
||||
from vllm.v1.simple_kv_offload.metadata import (
|
||||
SimpleCPUOffloadMetadata,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
SimpleCPUOffloadPromMetrics,
|
||||
SimpleCPUOffloadStats,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.worker import (
|
||||
SimpleCPUOffloadWorker,
|
||||
)
|
||||
@@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
hash_block_size=hash_block_size,
|
||||
lazy_offload=lazy_offload,
|
||||
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
|
||||
use_page_cache=use_page_cache if disk_mode else False,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.worker_handler = SimpleCPUOffloadWorker(
|
||||
@@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return self.scheduler_manager.get_boundary_store_stats()
|
||||
return None
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
# Worker-side callers may invoke this hook on connectors constructed
|
||||
# via __new__ (as in test_worker.py), which lack scheduler_manager.
|
||||
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
|
||||
self, "scheduler_manager", None
|
||||
)
|
||||
if scheduler_manager is not None:
|
||||
return scheduler_manager.get_stats()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
SimpleCPUOffloadStats(data=data)
|
||||
if data is not None
|
||||
else SimpleCPUOffloadStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return SimpleCPUOffloadPromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.take_events()
|
||||
|
||||
@@ -1583,6 +1583,14 @@ def get_dcp_group() -> GroupCoordinator:
|
||||
return _DCP
|
||||
|
||||
|
||||
def get_dcp_world_size_and_rank(enabled: bool = True) -> tuple[int, int]:
|
||||
"""Return ``(world_size, rank)`` in the DCP group, or ``(1, 0)`` when disabled
|
||||
(e.g. a replicated draft cache) or the group is uninitialized (unit tests)."""
|
||||
if not enabled or _DCP is None:
|
||||
return 1, 0
|
||||
return _DCP.world_size, _DCP.rank_in_group
|
||||
|
||||
|
||||
_PP: GroupCoordinator | None = None
|
||||
|
||||
|
||||
|
||||
+8
-13
@@ -141,7 +141,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
|
||||
VLLM_ROCM_USE_AITER_MOE: bool = True
|
||||
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto"
|
||||
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
|
||||
VLLM_ROCM_USE_AITER_MLA: bool = True
|
||||
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
|
||||
@@ -1280,18 +1280,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ROCM_USE_AITER_MOE": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4
|
||||
# activations, separated gate/up layout) instead of default a16w4. vLLM
|
||||
# sets AITER_SITUV2_A4W4 at init when this flag is on and clears any
|
||||
# legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first).
|
||||
# VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing
|
||||
# recipes; it does not select a8w4 kernels.
|
||||
# Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag
|
||||
# and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics.
|
||||
"VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1")
|
||||
or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower()
|
||||
in ("true", "1")
|
||||
# Activation dtype for the Kimi-K3 SiTU MXFP4 MoE (AITER FlyDSL SiTUv2):
|
||||
# auto (= a4w4), a4w4, a8w4 or a16w4. Legacy 1/0 mean a4w4/a16w4.
|
||||
"VLLM_ROCM_USE_AITER_MOE_SITUV2": env_with_choices(
|
||||
"VLLM_ROCM_USE_AITER_MOE_SITUV2",
|
||||
"auto",
|
||||
["auto", "a4w4", "a8w4", "a16w4", "0", "1"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
# MoE sorting dispatch policy for AITER fused MoE kernels.
|
||||
# 0 = auto (default): single-pass for small batches, multi-pass
|
||||
|
||||
@@ -385,10 +385,11 @@ def rocm_aiter_fused_experts(
|
||||
|
||||
gate_mode = ""
|
||||
if activation == MoEActivation.SITU:
|
||||
# SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4
|
||||
# fp4 activations with separated gate/up weights (AITER #4463);
|
||||
# default a16w4 SiTU also stays separated.
|
||||
gate_mode = GateMode.SEPARATED.value
|
||||
gate_mode = (
|
||||
GateMode.INTERLEAVE.value
|
||||
if rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
|
||||
else GateMode.SEPARATED.value
|
||||
)
|
||||
elif quant_config.use_mxfp4_w4a16:
|
||||
gate_mode = GateMode.INTERLEAVE.value
|
||||
elif activation_interleave is not None:
|
||||
|
||||
@@ -1596,8 +1596,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
|
||||
fp4_dtype = torch.float4_e2m1fn_x2
|
||||
e8m0_dtype = torch.float8_e8m0fnu
|
||||
# SiTUv2 flydsl uses separated gate/up layout (a4w4).
|
||||
guinterleave = False
|
||||
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
|
||||
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
w13_weight.data.view(fp4_dtype), 16, guinterleave
|
||||
)
|
||||
|
||||
@@ -697,7 +697,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
else:
|
||||
gemm_rs.warn_incompatible_projection()
|
||||
|
||||
def _wo_b_proj(self, z: torch.Tensor) -> torch.Tensor:
|
||||
def _wo_b_proj(self, z: torch.Tensor | QuantizedActivation) -> torch.Tensor:
|
||||
"""Apply ``wo_b``; with GEMM-RS bound, also reduce-scatter the result.
|
||||
|
||||
Every ``_o_proj`` implementation projects through this so the decoder
|
||||
@@ -707,7 +707,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
"""
|
||||
if self.gemm_rs is None:
|
||||
return self.wo_b(z)
|
||||
if self.gemm_rs.should_run(z):
|
||||
if isinstance(z, torch.Tensor) and self.gemm_rs.should_run(z):
|
||||
return self.gemm_rs.apply(z, self.wo_b)
|
||||
# Small batches stay on the unfused path, which is faster there.
|
||||
return sp_reduce_scatter(self.wo_b(z))
|
||||
|
||||
@@ -9,15 +9,16 @@ import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
|
||||
compute_fp8_einsum_recipe,
|
||||
deep_gemm_fp8_o_proj,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v41.common.ops import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
compute_global_topk_indices_and_lens,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia.ops.o_proj import (
|
||||
dsv41_o_proj,
|
||||
register_dsv41_o_proj_warmup,
|
||||
)
|
||||
from vllm.models.deepseek_v41.sparse_mla import (
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
DeepseekV4SparseMLABackend,
|
||||
@@ -242,27 +243,14 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
return _pad_to_supported_q_heads(num_heads)
|
||||
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.wo_a,
|
||||
self._wo_b_proj,
|
||||
n_groups=self.n_local_groups,
|
||||
heads_per_group=self.n_local_heads // self.n_local_groups,
|
||||
nope_dim=self.nope_head_dim,
|
||||
rope_dim=self.rope_head_dim,
|
||||
o_lora_rank=self.o_lora_rank,
|
||||
einsum_recipe=self._einsum_recipe,
|
||||
tma_aligned_scales=self._tma_aligned_scales,
|
||||
)
|
||||
return dsv41_o_proj(self, attn_out, positions)
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe(
|
||||
self._o_proj_block_size
|
||||
)
|
||||
register_dsv41_o_proj_warmup(self)
|
||||
# Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the
|
||||
# per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``.
|
||||
if self.kv_cache_torch_dtype != torch.float8_e4m3fn:
|
||||
@@ -623,21 +611,7 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
|
||||
return _pad_to_supported_q_heads(num_heads)
|
||||
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.wo_a,
|
||||
self._wo_b_proj,
|
||||
n_groups=self.n_local_groups,
|
||||
heads_per_group=self.n_local_heads // self.n_local_groups,
|
||||
nope_dim=self.nope_head_dim,
|
||||
rope_dim=self.rope_head_dim,
|
||||
o_lora_rank=self.o_lora_rank,
|
||||
einsum_recipe=self._einsum_recipe,
|
||||
tma_aligned_scales=self._tma_aligned_scales,
|
||||
)
|
||||
return dsv41_o_proj(self, attn_out, positions)
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
|
||||
super().__init__(vllm_config, *args, **kwargs)
|
||||
|
||||
@@ -6,16 +6,17 @@ from typing import TYPE_CHECKING, ClassVar, cast
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
|
||||
compute_fp8_einsum_recipe,
|
||||
deep_gemm_fp8_o_proj,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v41.common.ops import (
|
||||
combine_topk_swa_indices,
|
||||
compute_global_topk_indices_and_lens,
|
||||
dequantize_and_gather_k_cache,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia.ops.o_proj import (
|
||||
dsv41_o_proj,
|
||||
register_dsv41_o_proj_warmup,
|
||||
)
|
||||
from vllm.models.deepseek_v41.sparse_mla import (
|
||||
DeepseekV4FlashMLABackend,
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
@@ -57,23 +58,10 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
|
||||
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe(
|
||||
self._o_proj_block_size
|
||||
)
|
||||
register_dsv41_o_proj_warmup(self)
|
||||
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.wo_a,
|
||||
self._wo_b_proj,
|
||||
n_groups=self.n_local_groups,
|
||||
heads_per_group=self.n_local_heads // self.n_local_groups,
|
||||
nope_dim=self.nope_head_dim,
|
||||
rope_dim=self.rope_head_dim,
|
||||
o_lora_rank=self.o_lora_rank,
|
||||
einsum_recipe=self._einsum_recipe,
|
||||
tma_aligned_scales=self._tma_aligned_scales,
|
||||
)
|
||||
return dsv41_o_proj(self, attn_out, positions)
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
|
||||
@@ -0,0 +1,599 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DSV4.1 small-batch MXFP8 WO-A with inverse RoPE and quantization."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import cutlass.utils as utils
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import (
|
||||
BFloat16,
|
||||
Float8E4M3FN,
|
||||
Float32,
|
||||
Int32,
|
||||
Int64,
|
||||
Uint8,
|
||||
Uint32,
|
||||
Uint64,
|
||||
)
|
||||
from cutlass._mlir.dialects import llvm
|
||||
from cutlass.cute.nvgpu import cpasync, tcgen05
|
||||
from cutlass.cute.runtime import make_fake_tensor
|
||||
from cutlass.cutlass_dsl import dsl_user_op
|
||||
|
||||
from vllm.cute_utils import _tcgen05, mbarrier, recast_val, simple_tma_copy
|
||||
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
)
|
||||
|
||||
# Each head is [nope | rope] as in fused_inv_rope_fp8_quant: only the last
|
||||
# _ROPE_DIM columns carry RoPE.
|
||||
_HEAD_DIM = 512
|
||||
_ROPE_DIM = 64
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def _ue8m0_rp(value, *, loc=None, ip=None):
|
||||
result = llvm.inline_asm(
|
||||
Uint32.mlir_type,
|
||||
[value.ir_value(loc=loc, ip=ip)],
|
||||
"{ .reg .b16 sf; cvt.rp.satfinite.ue8m0x2.f32 sf, $1, $1; "
|
||||
"cvt.u32.u16 $0, sf; and.b32 $0, $0, 255; }",
|
||||
"=r,f",
|
||||
has_side_effects=False,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return Uint32(result)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def _rope_fma(value, cos, other, sin, *, subtract=False, loc=None, ip=None):
|
||||
"""Preserve multiply/FMA rounding when RoPE loads are hoisted."""
|
||||
result = llvm.inline_asm(
|
||||
Float32.mlir_type,
|
||||
[v.ir_value(loc=loc, ip=ip) for v in (value, cos, other, sin)],
|
||||
"{ .reg .f32 product; mul.rn.f32 product, $3, $4; "
|
||||
+ (
|
||||
"neg.f32 product, product; fma.rn.f32 $0, $1, $2, product; }"
|
||||
if subtract
|
||||
else "fma.rn.f32 $0, $1, $2, product; }"
|
||||
),
|
||||
"=f,f,f,f,f",
|
||||
has_side_effects=False,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return Float32(result)
|
||||
|
||||
|
||||
@cute.jit
|
||||
def _scale(amax: Float32):
|
||||
exponent = _ue8m0_rp(amax * Float32(1.0 / 448.0))
|
||||
inv = recast_val((Uint32(254) - exponent) << 23, Float32)
|
||||
if exponent == 0:
|
||||
inv = Float32(0)
|
||||
return exponent, inv
|
||||
|
||||
|
||||
class FusedWoAKernel(VllmCuTeDSLJitKernel["FusedWoAKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
# T=1..96 uses at most 46 keys per (n_groups, heads_per_group, o_lora_rank).
|
||||
tokens_per_tile: int
|
||||
token_tiles: int
|
||||
full_tiles: bool
|
||||
n_groups: int
|
||||
heads_per_group: int
|
||||
o_lora_rank: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
tokens = compile_key.tokens_per_tile
|
||||
token_tiles = compile_key.token_tiles
|
||||
full_tiles = compile_key.full_tiles
|
||||
acc_cols = 1 << (tokens - 1).bit_length()
|
||||
tile_m = max(8, acc_cols)
|
||||
tile_n = 128
|
||||
tile_k = 128
|
||||
# tcgen05 MXFP8 MMA consumes K = 32, one MX block, per instruction.
|
||||
mma_k = 32
|
||||
# Each group's heads split K across one cluster, one head per CTA.
|
||||
heads_per_group = compile_key.heads_per_group
|
||||
tiles_per_group = compile_key.o_lora_rank // tile_n
|
||||
n_tiles = compile_key.n_groups * tiles_per_group
|
||||
# Each CTA keeps one whole head of K in SMEM, one TMA box per stage.
|
||||
num_stages = _HEAD_DIM // tile_k
|
||||
w_stage_bytes = tile_n * tile_k
|
||||
nope_dim = _HEAD_DIM - _ROPE_DIM
|
||||
threads_per_token = _HEAD_DIM // 4
|
||||
sf_col = max(16, tile_m)
|
||||
tmem_cols = sf_col * 2
|
||||
threads = min(acc_cols * threads_per_token, 512)
|
||||
tokens_per_iter = threads // threads_per_token
|
||||
|
||||
@cute.kernel
|
||||
def device_kernel(
|
||||
x: cute.Tensor,
|
||||
positions: cute.Tensor,
|
||||
rope: cute.Tensor,
|
||||
w: cpasync.TmaInfo,
|
||||
ws: cpasync.TmaInfo,
|
||||
q: cute.Tensor,
|
||||
qs: cute.Tensor,
|
||||
num_tokens: Int32,
|
||||
):
|
||||
total_tokens = num_tokens
|
||||
if cutlass.const_expr(full_tiles):
|
||||
total_tokens = Int32(tokens * token_tiles)
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
bid, token_tile, _ = cute.arch.block_idx()
|
||||
token_start = Int32(0)
|
||||
if cutlass.const_expr(token_tiles > 1):
|
||||
token_start = token_tile * tokens
|
||||
warp = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane = tid % 32
|
||||
split = bid % heads_per_group
|
||||
tile = bid // heads_per_group
|
||||
group = tile // tiles_per_group
|
||||
smem = utils.SmemAllocator()
|
||||
sW = smem.allocate_tensor(
|
||||
Float8E4M3FN,
|
||||
w.smem_layout.outer,
|
||||
byte_alignment=128,
|
||||
swizzle=w.smem_layout.inner,
|
||||
)
|
||||
sX = smem.allocate_tensor(
|
||||
Float8E4M3FN,
|
||||
cute.make_layout(
|
||||
(tile_m, tile_k, num_stages), stride=(tile_k, 1, tile_m * tile_k)
|
||||
),
|
||||
byte_alignment=1024,
|
||||
swizzle=cute.make_swizzle(3, 4, 3),
|
||||
)
|
||||
# tcgen05.cp 32x128b.warpx4 order: row r at byte (r % 32) * 16 +
|
||||
# (r // 32) * 4, one 4-byte cell of MX-block scales per row.
|
||||
sW_SF = smem.allocate_tensor(
|
||||
Int32,
|
||||
cute.make_layout(((32, 4), num_stages), stride=((4, 1), tile_n)),
|
||||
128,
|
||||
)
|
||||
sW_SF_raw = smem.allocate_tensor(
|
||||
Int32, cute.make_layout((tile_n, num_stages)), 128
|
||||
)
|
||||
sX_SF = smem.allocate_tensor(
|
||||
Uint8,
|
||||
cute.make_layout(
|
||||
((32, 4), tile_k // mma_k, num_stages),
|
||||
stride=((16, 4), 1, tile_n * 4),
|
||||
),
|
||||
128,
|
||||
)
|
||||
copy_fp8x4 = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyUniversalOp(), Float8E4M3FN, num_bits_per_copy=32
|
||||
)
|
||||
partial = smem.allocate_tensor(
|
||||
Float32,
|
||||
cute.make_layout(
|
||||
(tile_n, cute.ceil_div(tokens, heads_per_group), heads_per_group)
|
||||
),
|
||||
128,
|
||||
)
|
||||
mbar_loaded = smem.allocate_array(Int64, num_stages)
|
||||
mbar_ws_loaded = smem.allocate_array(Int64, 1)
|
||||
mbar_done = smem.allocate_array(Int64, 1)
|
||||
mbar_reduced = smem.allocate_array(Int64, 1)
|
||||
taddr = smem.allocate(Int32, 4)
|
||||
if tid == 0:
|
||||
for stage in cutlass.range_constexpr(num_stages):
|
||||
# Wait for weight TMA and this stage's quantization threads.
|
||||
cute.arch.mbarrier_init(mbar_loaded + stage, 1 + threads // 4)
|
||||
cute.arch.mbarrier_init(mbar_ws_loaded, 1)
|
||||
cute.arch.mbarrier_init(mbar_done, 1)
|
||||
cute.arch.mbarrier_init(mbar_reduced, 1)
|
||||
cute.arch.mbarrier_init_fence()
|
||||
# Local users of the barriers (warp 0's TMA below) need the init too.
|
||||
cute.arch.sync_threads()
|
||||
# Paired with cluster_wait() below: peers must see mbar_reduced
|
||||
# initialized before their st.async; the split hides the barrier.
|
||||
cute.arch.cluster_arrive_relaxed()
|
||||
if warp == 0:
|
||||
# Raw DeepGEMM MN-major scales; the MMA warp transposes them to
|
||||
# the UTCCP order, which TMA cannot scatter at 4-byte granularity.
|
||||
ws_tiles = cute.zipped_divide(ws.tma_tensor, (tile_n, num_stages))
|
||||
with cute.arch.elect_one():
|
||||
mbarrier.arrive_expect_tx(mbar_ws_loaded, tile_n * num_stages * 4)
|
||||
simple_tma_copy(
|
||||
ws.atom,
|
||||
ws_tiles[None, (tile % tiles_per_group, split, group)],
|
||||
sW_SF_raw,
|
||||
mbar_ws_loaded,
|
||||
)
|
||||
tiles = cute.zipped_divide(w.tma_tensor, (tile_n, tile_k))
|
||||
for stage in cutlass.range_constexpr(num_stages):
|
||||
with cute.arch.elect_one():
|
||||
mbarrier.arrive_expect_tx(mbar_loaded + stage, w_stage_bytes)
|
||||
simple_tma_copy(
|
||||
w.atom,
|
||||
tiles[None, (tile, split * num_stages + stage)],
|
||||
sW[None, None, stage],
|
||||
mbar_loaded + stage,
|
||||
)
|
||||
# x and positions come from the PDL predecessor; everything above
|
||||
# only touches weights. Padded sX rows only feed unread MMA columns.
|
||||
cute.arch.griddepcontrol_wait()
|
||||
# TMEM is allocated at runtime, and a PDL predecessor on this SM may
|
||||
# hold it until it exits; allocating earlier can deadlock.
|
||||
if warp == 0:
|
||||
cute.arch.alloc_tmem(tmem_cols, taddr)
|
||||
cute.arch.relinquish_tmem_alloc_permit()
|
||||
copy_x = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=64
|
||||
)
|
||||
n_iters = cute.ceil_div(tokens, tokens_per_iter)
|
||||
qid = (tid + 32) % threads_per_token
|
||||
k = qid * 4
|
||||
# Tail rows repeat the last input; their output stores are masked.
|
||||
# Issue every token's x load before any math so their latencies overlap.
|
||||
x_bf16 = cute.make_rmem_tensor((4, n_iters), BFloat16)
|
||||
for iteration in cutlass.range_constexpr(n_iters):
|
||||
token = iteration * tokens_per_iter + tid // threads_per_token
|
||||
if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens:
|
||||
x_src = cute.local_tile(
|
||||
x[
|
||||
cute.min(token_start + token, total_tokens - 1),
|
||||
group * heads_per_group + split,
|
||||
None,
|
||||
],
|
||||
(4,),
|
||||
(qid,),
|
||||
)
|
||||
cute.copy(copy_x, x_src, x_bf16[None, iteration])
|
||||
# Overlap the RoPE loads across tokens before consuming any of them.
|
||||
cos_reg = cute.make_rmem_tensor((2, n_iters), Float32)
|
||||
sin_reg = cute.make_rmem_tensor((2, n_iters), Float32)
|
||||
if cutlass.const_expr(n_iters > 1):
|
||||
copy_rope = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=32
|
||||
)
|
||||
if k >= nope_dim:
|
||||
for iteration in cutlass.range_constexpr(n_iters):
|
||||
token = iteration * tokens_per_iter + tid // threads_per_token
|
||||
if (
|
||||
cutlass.const_expr(tokens % tokens_per_iter == 0)
|
||||
or token < tokens
|
||||
):
|
||||
src_token = cute.min(token_start + token, total_tokens - 1)
|
||||
pos = positions[src_token]
|
||||
freq = (k - nope_dim) // 2
|
||||
rope_row = rope[pos, None]
|
||||
cos_src = cute.local_tile(rope_row, (2,), (freq // 2,))
|
||||
cute.copy(copy_rope, cos_src, cos_reg[None, iteration])
|
||||
sin_tile = (freq + _ROPE_DIM // 2) // 2
|
||||
sin_src = cute.local_tile(rope_row, (2,), (sin_tile,))
|
||||
cute.copy(copy_rope, sin_src, sin_reg[None, iteration])
|
||||
for iteration in cutlass.range_constexpr(n_iters):
|
||||
token = iteration * tokens_per_iter + tid // threads_per_token
|
||||
if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens:
|
||||
x_f32 = cute.make_rmem_tensor((4,), Float32)
|
||||
x_f32.store(x_bf16[None, iteration].load().to(Float32))
|
||||
if k >= nope_dim:
|
||||
# Inverse RoPE on the interleaved pairs (k, k+1) and
|
||||
# (k+2, k+3); each rope row is cos || sin.
|
||||
for pair in cutlass.range_constexpr(2):
|
||||
if cutlass.const_expr(n_iters > 1):
|
||||
cos = cos_reg[pair, iteration]
|
||||
sin = sin_reg[pair, iteration]
|
||||
else:
|
||||
pos = positions[
|
||||
cute.min(token_start + token, total_tokens - 1)
|
||||
]
|
||||
freq = (k - nope_dim) // 2 + pair
|
||||
cos = Float32(rope[pos, freq])
|
||||
sin = Float32(rope[pos, freq + _ROPE_DIM // 2])
|
||||
even, odd = x_f32[2 * pair], x_f32[2 * pair + 1]
|
||||
x_f32[2 * pair] = _rope_fma(even, cos, odd, sin)
|
||||
x_f32[2 * pair + 1] = _rope_fma(
|
||||
odd, cos, even, sin, subtract=True
|
||||
)
|
||||
amax = cute.arch.fmax(
|
||||
cute.arch.fmax(cute.abs(x_f32[0]), cute.abs(x_f32[1])),
|
||||
cute.arch.fmax(cute.abs(x_f32[2]), cute.abs(x_f32[3])),
|
||||
)
|
||||
amax = cute.arch.warp_reduction_max(amax, threads_in_group=8)
|
||||
exponent, inv = _scale(cute.arch.fmax(amax, Float32(1e-10)))
|
||||
x_fp8 = cute.make_rmem_tensor((4,), Float8E4M3FN)
|
||||
x_fp8.store((x_f32.load() * inv).to(Float8E4M3FN))
|
||||
cute.copy(
|
||||
copy_fp8x4,
|
||||
x_fp8,
|
||||
cute.local_tile(
|
||||
sX[token, None, k // tile_k], (4,), (qid % 32,)
|
||||
),
|
||||
)
|
||||
if lane % 8 == 0:
|
||||
sX_SF[token, k % tile_k // mma_k, k // tile_k] = Uint8(exponent)
|
||||
cute.arch.fence_proxy("async.shared", space="cta")
|
||||
mbarrier.arrive(mbar_loaded + qid // 32, order="release")
|
||||
if warp == 0:
|
||||
base = cute.make_tensor(taddr, cute.make_layout(1))[0]
|
||||
# sW / sX are 128B-swizzled K-major; one stage's K fills the
|
||||
# swizzle atom, so the leading byte offset is unused.
|
||||
sdesc = _tcgen05.make_sdesc_128B_swizzle(LBO=0)
|
||||
# Unswizzled 32 x 16 B scale tiles: SBO = one 8-row core matrix
|
||||
# (16 B units at bit 32); bit 46 is the SM100 descriptor version.
|
||||
sf_sbo = 8 * 16
|
||||
sfdesc = Uint64((sf_sbo >> 4 << 32) | (1 << 46))
|
||||
# M = tile_n weight rows (A), N = tile_m token rows (B).
|
||||
idesc = _tcgen05.make_mxfp8_idesc(tile_n, tile_m)
|
||||
# sW_SF's layout is the UTCCP order, so a plain copy transposes;
|
||||
# fence the generic-proxy writes before tcgen05.cp reads them.
|
||||
cute.arch.mbarrier_wait(mbar_ws_loaded, 0)
|
||||
for stage in cutlass.range_constexpr(num_stages):
|
||||
for i in cutlass.range_constexpr(4):
|
||||
sW_SF[i * 32 + lane, stage] = sW_SF_raw[i * 32 + lane, stage]
|
||||
cute.arch.fence_proxy("async.shared", space="cta")
|
||||
cute.arch.sync_warp()
|
||||
for stage in cutlass.range_constexpr(num_stages):
|
||||
cute.arch.mbarrier_wait(mbar_loaded + stage, 0)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
if cutlass.const_expr(stage == num_stages - 1):
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
adesc = sdesc | (sW[None, None, stage].iterator.toint() >> 4)
|
||||
bdesc = sdesc | (sX[None, None, stage].iterator.toint() >> 4)
|
||||
_tcgen05.cp(
|
||||
base + sf_col,
|
||||
sfdesc | (sW_SF[None, stage].iterator.toint() >> 4),
|
||||
"32x128b",
|
||||
"warpx4",
|
||||
)
|
||||
_tcgen05.cp(
|
||||
base + sf_col + 4,
|
||||
sfdesc | (sX_SF[None, None, stage].iterator.toint() >> 4),
|
||||
"32x128b",
|
||||
"warpx4",
|
||||
)
|
||||
# Advance mma_k FP8 bytes (16 B units) and pick MX block kk of
|
||||
# each 4-byte scale (B sf_id at bit 4, A sf_id at bit 29).
|
||||
for kk in cutlass.range_constexpr(tile_k // mma_k):
|
||||
_tcgen05.mma_mxfp8(
|
||||
base,
|
||||
adesc + kk * mma_k // 16,
|
||||
bdesc + kk * mma_k // 16,
|
||||
idesc + ((kk << 4) | (kk << 29)),
|
||||
base + sf_col,
|
||||
base + sf_col + 4,
|
||||
stage > 0 or kk > 0,
|
||||
)
|
||||
_tcgen05.commit(mbar_done)
|
||||
# Only the MMA warp waits; barrier 2 releases the TMEM readers.
|
||||
cute.arch.mbarrier_wait(mbar_done, 0)
|
||||
if tid < tile_n:
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=tile_n)
|
||||
# Tokens t with t % heads_per_group == split arrive from every peer
|
||||
# (this CTA included), tile_n Float32 rows each.
|
||||
if tid == 0:
|
||||
owned = cute.ceil_div(tokens - split, heads_per_group)
|
||||
mbarrier.arrive_expect_tx(
|
||||
mbar_reduced, owned * tile_n * heads_per_group * 4
|
||||
)
|
||||
# Pairs with cluster_arrive_relaxed() above: every peer's
|
||||
# mbar_reduced is initialized before the st.async below.
|
||||
cute.arch.cluster_wait()
|
||||
# quack's process-wide const_expr-if rewrite can leave these unbound
|
||||
# after the dynamic path above; bind them so the regions join.
|
||||
amax, exponent, inv = Float32(0), Uint32(0), Float32(0)
|
||||
if tid < tile_n:
|
||||
base = cute.make_tensor(taddr, cute.make_layout(1))[0]
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
acc = cute.make_rmem_tensor(acc_cols, Float32)
|
||||
if cutlass.const_expr(tokens == 1):
|
||||
acc[0] = _tcgen05.ld(warp * 32, base, "32x32b", 1)
|
||||
else:
|
||||
acc.store(_tcgen05.ld(warp * 32, base, "32x32b", acc_cols))
|
||||
_tcgen05.wait_ld()
|
||||
for token in cutlass.range_constexpr(tokens):
|
||||
ptr = cute.domain_offset(
|
||||
(tid, token // heads_per_group, split), partial
|
||||
).iterator
|
||||
cute.arch.store_async_dsmem(
|
||||
ptr,
|
||||
recast_val(acc[token], Int32),
|
||||
mbar_reduced,
|
||||
token % heads_per_group,
|
||||
)
|
||||
if split < tokens:
|
||||
# One warp waits for the peers' bytes; named barrier 1 (0 is
|
||||
# sync_threads) releases the other tile_n epilogue threads.
|
||||
if warp == 0:
|
||||
cute.arch.mbarrier_wait(mbar_reduced, 0)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=tile_n)
|
||||
for i in cutlass.range_constexpr(
|
||||
cute.ceil_div(tokens, heads_per_group)
|
||||
):
|
||||
token = split + i * heads_per_group
|
||||
if cutlass.const_expr(
|
||||
full_tiles
|
||||
and (
|
||||
tokens <= heads_per_group
|
||||
or tokens % heads_per_group == 0
|
||||
)
|
||||
) or (token < tokens and token_start + token < total_tokens):
|
||||
value = Float32(0)
|
||||
for peer in cutlass.range_constexpr(heads_per_group):
|
||||
value += partial[tid, i, peer]
|
||||
value = Float32(BFloat16(value))
|
||||
amax = cute.arch.warp_redux_sync(value, "max", abs=True)
|
||||
exponent, inv = _scale(amax)
|
||||
q[token_start + token, tile * tile_n + tid] = Float8E4M3FN(
|
||||
value * inv
|
||||
)
|
||||
if lane == 0:
|
||||
row = token_start + token
|
||||
offset = row % 32 * 16 + row // 32 * 4 + warp
|
||||
qs[tile * 512 + offset] = Uint8(exponent)
|
||||
# Every scale tile owns its padding, with no second memset kernel.
|
||||
for i in cutlass.range_constexpr(4):
|
||||
offset = i * tile_n + tid
|
||||
row = offset // 16 + (offset % 16 // 4) * 32
|
||||
if token_tile == 0 and split == 0 and row >= total_tokens:
|
||||
qs[tile * 512 + offset] = Uint8(0)
|
||||
# Each receiver drained all writes into its own inbox. Only local
|
||||
# TMEM readers must finish before deallocation; no peer reads SMEM.
|
||||
if tid < tile_n:
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=tile_n)
|
||||
if warp == 0:
|
||||
cute.arch.dealloc_tmem(
|
||||
cute.make_ptr(
|
||||
Float32,
|
||||
cute.make_tensor(taddr, cute.make_layout(1))[0],
|
||||
cute.AddressSpace.tmem,
|
||||
),
|
||||
tmem_cols,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def host_entrypoint(
|
||||
x: cute.Tensor,
|
||||
positions: cute.Tensor,
|
||||
rope: cute.Tensor,
|
||||
w: cute.Tensor,
|
||||
ws: cute.Tensor,
|
||||
q: cute.Tensor,
|
||||
qs: cute.Tensor,
|
||||
num_tokens: Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
layout = cute.make_composed_layout(
|
||||
cute.make_swizzle(3, 4, 3),
|
||||
0,
|
||||
cute.make_layout(
|
||||
(tile_n, tile_k, num_stages), stride=(tile_k, 1, w_stage_bytes)
|
||||
),
|
||||
)
|
||||
tma = cpasync.make_tiled_tma_atom(
|
||||
cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE),
|
||||
w,
|
||||
layout,
|
||||
(tile_n, tile_k),
|
||||
)
|
||||
ws_tma = cpasync.make_tiled_tma_atom(
|
||||
cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE),
|
||||
cute.make_tensor(ws.iterator, cute.select(ws.layout, mode=[1, 2, 0])),
|
||||
cute.make_layout((tile_n, num_stages)),
|
||||
(tile_n, num_stages),
|
||||
)
|
||||
device_kernel(x, positions, rope, tma, ws_tma, q, qs, num_tokens).launch(
|
||||
grid=(n_tiles * heads_per_group, token_tiles, 1),
|
||||
block=(threads, 1, 1),
|
||||
cluster=(heads_per_group, 1, 1),
|
||||
stream=stream,
|
||||
use_pdl=True,
|
||||
)
|
||||
|
||||
return host_entrypoint
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self, *, tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int
|
||||
) -> CompileKey:
|
||||
token_tiles = (tokens + 31) // 32
|
||||
# Host integer arithmetic: balance tiles in multiples of four tokens.
|
||||
tile_tokens = (
|
||||
tokens
|
||||
if token_tiles == 1
|
||||
else (tokens + 4 * token_tiles - 1) // (4 * token_tiles) * 4
|
||||
)
|
||||
return self.CompileKey(
|
||||
tokens_per_tile=tile_tokens,
|
||||
token_tiles=token_tiles,
|
||||
full_tiles=tokens % tile_tokens == 0,
|
||||
n_groups=n_groups,
|
||||
heads_per_group=heads_per_group,
|
||||
o_lora_rank=o_lora_rank,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self, *, max_tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int
|
||||
) -> list[CompileKey]:
|
||||
# Trace all accepted token counts and deduplicate shared tile shapes.
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
tokens=WarmupIntRange(1, max_tokens + 1),
|
||||
n_groups=n_groups,
|
||||
heads_per_group=heads_per_group,
|
||||
o_lora_rank=o_lora_rank,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
|
||||
tokens = (
|
||||
compile_key.tokens_per_tile * compile_key.token_tiles
|
||||
if compile_key.full_tiles
|
||||
else cute.sym_int()
|
||||
)
|
||||
groups = compile_key.n_groups
|
||||
rank = compile_key.o_lora_rank
|
||||
n = groups * rank
|
||||
k = compile_key.heads_per_group * _HEAD_DIM
|
||||
return (
|
||||
make_fake_tensor(
|
||||
BFloat16,
|
||||
(tokens, groups * compile_key.heads_per_group, _HEAD_DIM),
|
||||
(cute.sym_int(divisibility=8), _HEAD_DIM, 1),
|
||||
assumed_align=16,
|
||||
),
|
||||
make_fake_tensor(Int64, (tokens,), (1,)),
|
||||
make_fake_tensor(Float32, (cute.sym_int(), _ROPE_DIM), (_ROPE_DIM, 1)),
|
||||
make_fake_tensor(Float8E4M3FN, (n, k), (k, 1), assumed_align=16),
|
||||
make_fake_tensor(
|
||||
Int32,
|
||||
(groups, rank, k // 128),
|
||||
(rank * k // 128, 1, rank),
|
||||
assumed_align=16,
|
||||
),
|
||||
make_fake_tensor(Float8E4M3FN, (tokens, n), (n, 1)),
|
||||
make_fake_tensor(Uint8, (128 * n // 32,), (1,)),
|
||||
Int32(0),
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rope: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> CuTeDSLLaunchSpec["FusedWoAKernel.CompileKey"]:
|
||||
"""Return FP8 WO-A output and FlashInfer F8_128x4 scale bytes."""
|
||||
tokens = x.shape[0]
|
||||
groups, rank = weight_scale.shape[:2]
|
||||
n = groups * rank
|
||||
q = torch.empty((tokens, n), device=x.device, dtype=torch.float8_e4m3fn)
|
||||
scales = torch.empty(128 * n // 32, device=x.device, dtype=torch.uint8)
|
||||
launch_args = (
|
||||
x,
|
||||
positions,
|
||||
rope,
|
||||
weight.view(n, -1),
|
||||
weight_scale,
|
||||
q,
|
||||
scales,
|
||||
tokens,
|
||||
)
|
||||
compile_key = self.dispatch(
|
||||
tokens=tokens,
|
||||
n_groups=groups,
|
||||
heads_per_group=x.shape[1] // groups,
|
||||
o_lora_rank=rank,
|
||||
)
|
||||
return compile_key, launch_args, (q, scales)
|
||||
|
||||
|
||||
_FUSED_WO_A_KERNEL = FusedWoAKernel()
|
||||
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DSV4.1 output projection with a small-batch SM100/SM103 fusion."""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _fused_wo_a_max_tokens(layer: nn.Module) -> int:
|
||||
max_tokens = getattr(layer, "_fused_wo_a_token_limit", None)
|
||||
if max_tokens is None:
|
||||
# Additional token tiles only pay off for smaller grids.
|
||||
ctas_per_tile = layer.n_local_heads * (layer.o_lora_rank // 128)
|
||||
max_tokens = 96 if ctas_per_tile <= 128 else 32
|
||||
layer._fused_wo_a_token_limit = max_tokens
|
||||
return max_tokens
|
||||
|
||||
|
||||
def _can_fuse_wo_a(layer: nn.Module) -> bool:
|
||||
"""Whether the attention layer matches the fused WO-A kernel's layout."""
|
||||
method = getattr(layer.wo_b, "scheme", layer.wo_b.quant_method)
|
||||
return (
|
||||
current_platform.is_device_capability_family(100)
|
||||
# WO-A is FP8 with per-32 scales, and WO-B takes MXFP8 input directly.
|
||||
and layer._einsum_recipe == (1, 1, 32)
|
||||
and get_input_quant_key(layer.wo_b) == kMxfp8Dynamic
|
||||
and isinstance(
|
||||
getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel
|
||||
)
|
||||
# A group's heads form one portable (<= 8 CTA) cluster.
|
||||
and layer.n_local_heads // layer.n_local_groups <= 8
|
||||
and (layer.nope_head_dim, layer.rope_head_dim) == (448, 64)
|
||||
and layer.o_lora_rank % 128 == 0
|
||||
)
|
||||
|
||||
|
||||
def register_dsv41_o_proj_warmup(layer: nn.Module) -> None:
|
||||
"""Warm the fused WO-A token counts ``dsv41_o_proj`` can dispatch."""
|
||||
# Draft models load outside the warmup registry; their layers share the
|
||||
# target's attention shapes, so the target's registration covers them too.
|
||||
if _can_fuse_wo_a(layer):
|
||||
from .fused_wo_a import _FUSED_WO_A_KERNEL
|
||||
|
||||
_FUSED_WO_A_KERNEL.register_warmup(
|
||||
max_tokens=_fused_wo_a_max_tokens(layer),
|
||||
n_groups=layer.n_local_groups,
|
||||
heads_per_group=layer.n_local_heads // layer.n_local_groups,
|
||||
o_lora_rank=layer.o_lora_rank,
|
||||
)
|
||||
|
||||
|
||||
def dsv41_o_proj(
|
||||
layer: nn.Module, attn_out: torch.Tensor, positions: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100/SM103 batches."""
|
||||
o = attn_out[:, : layer.n_local_heads, :]
|
||||
cos_sin_cache = layer.rotary_emb.cos_sin_cache
|
||||
if 1 <= o.shape[0] <= _fused_wo_a_max_tokens(layer) and _can_fuse_wo_a(layer):
|
||||
from .fused_wo_a import _FUSED_WO_A_KERNEL
|
||||
|
||||
q, scales = _FUSED_WO_A_KERNEL(
|
||||
x=o,
|
||||
positions=positions,
|
||||
rope=cos_sin_cache,
|
||||
weight=layer.wo_a.weight,
|
||||
weight_scale=layer.wo_a.weight_scale,
|
||||
)
|
||||
return layer._wo_b_proj(
|
||||
QuantizedActivation(q, scales, o.dtype, q.shape, kMxfp8Dynamic)
|
||||
)
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
layer.wo_a,
|
||||
layer._wo_b_proj,
|
||||
n_groups=layer.n_local_groups,
|
||||
heads_per_group=layer.n_local_heads // layer.n_local_groups,
|
||||
nope_dim=layer.nope_head_dim,
|
||||
rope_dim=layer.rope_head_dim,
|
||||
o_lora_rank=layer.o_lora_rank,
|
||||
einsum_recipe=layer._einsum_recipe,
|
||||
tma_aligned_scales=layer._tma_aligned_scales,
|
||||
)
|
||||
@@ -383,6 +383,7 @@ def _prefill_logits(
|
||||
page_table: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
visible_blocks: torch.Tensor,
|
||||
logits_workspace: torch.Tensor,
|
||||
max_query_len: int,
|
||||
logits_width: int,
|
||||
query_offset: int,
|
||||
@@ -392,10 +393,14 @@ def _prefill_logits(
|
||||
assert visible_blocks.shape == (q.shape[0],)
|
||||
assert 0 <= query_offset <= query_offset + num_queries <= q.shape[0]
|
||||
assert 0 < logits_width <= page_table.shape[1] * k_cache.shape[1]
|
||||
assert logits_workspace.is_contiguous()
|
||||
assert logits_workspace.numel() >= num_queries * logits_width
|
||||
|
||||
logits = torch.empty(
|
||||
(num_queries, logits_width), dtype=torch.float32, device=q.device
|
||||
# slice from pre-allocated workspace
|
||||
logits = logits_workspace[: num_queries * logits_width].view(
|
||||
num_queries, logits_width
|
||||
)
|
||||
|
||||
# tuned on GB300
|
||||
if k_cache.dtype == torch.float8_e4m3fn:
|
||||
TILE_R, STAGES, num_warps = 32, 2, 8
|
||||
@@ -610,8 +615,13 @@ def qsa_select_paged_prefill(
|
||||
logits_width = min(max(64, logits_width), page_table.shape[1] * k_cache.shape[1])
|
||||
|
||||
# chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB
|
||||
# always allocate the worst case to avoid memory fragmentation.
|
||||
max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
|
||||
rows_per_chunk = max(1, max_logits_bytes // (logits_width * 4))
|
||||
|
||||
budget_bytes = max(max_logits_bytes, page_table.shape[1] * k_cache.shape[1] * 4)
|
||||
logits_workspace = q.new_empty(budget_bytes // 4, dtype=torch.float32)
|
||||
|
||||
topk_workspace = torch.empty(
|
||||
(_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device
|
||||
)
|
||||
@@ -625,6 +635,7 @@ def qsa_select_paged_prefill(
|
||||
page_table,
|
||||
query_start_loc,
|
||||
visible_blocks,
|
||||
logits_workspace,
|
||||
max_query_len,
|
||||
logits_width,
|
||||
query_offset=query_start,
|
||||
|
||||
@@ -870,15 +870,14 @@ class AttentionImplBase(ABC, Generic[T]):
|
||||
def __new__(cls, *args, **kwargs):
|
||||
# use __new__ so that all subclasses will call this
|
||||
self = super().__new__(cls)
|
||||
try:
|
||||
from vllm.distributed.parallel_state import get_dcp_group
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.distributed.parallel_state import get_dcp_world_size_and_rank
|
||||
|
||||
self.dcp_world_size = get_dcp_group().world_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group
|
||||
except AssertionError:
|
||||
# DCP might not be initialized in testing
|
||||
self.dcp_world_size = 1
|
||||
self.dcp_rank = 0
|
||||
# Replicated drafts run at DCP=1 inside a DCP target's process group.
|
||||
config = get_current_vllm_config_or_none()
|
||||
self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank(
|
||||
config is None or config.parallel_config.decode_context_parallel_size > 1
|
||||
)
|
||||
try:
|
||||
from vllm.distributed.parallel_state import get_pcp_group
|
||||
|
||||
|
||||
@@ -60,7 +60,10 @@ from vllm.config import (
|
||||
get_layers_from_vllm_config,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed.parallel_state import get_dcp_group
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dcp_group,
|
||||
get_dcp_world_size_and_rank,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
@@ -726,15 +729,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
num_qo_heads=self.num_heads_q,
|
||||
)
|
||||
|
||||
try:
|
||||
from vllm.distributed.parallel_state import get_dcp_group
|
||||
|
||||
self.dcp_world_size = get_dcp_group().world_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group
|
||||
except AssertionError:
|
||||
# DCP might not be initialized in testing
|
||||
self.dcp_world_size = 1
|
||||
self.dcp_rank = 0
|
||||
self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank(
|
||||
kv_cache_spec.dcp_sharded
|
||||
)
|
||||
|
||||
# Fused draft decode reuses the captured metadata object across draft
|
||||
# steps. For DCP, build-time host-side decisions such as
|
||||
|
||||
@@ -30,7 +30,10 @@ from vllm.config import (
|
||||
get_current_vllm_config_or_none,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed.parallel_state import get_dcp_group
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dcp_group,
|
||||
get_dcp_world_size_and_rank,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -641,6 +644,7 @@ class FlashInferTrtllmAPIDecode:
|
||||
|
||||
@dataclass
|
||||
class FlashInferMetadata:
|
||||
kv_cache_layout: KVCacheLayout
|
||||
num_actual_tokens: int
|
||||
"""Total number of tokens in the batch (excluding padding)."""
|
||||
|
||||
@@ -747,17 +751,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
self._decode_cudagraph_max_bs,
|
||||
self.compilation_config.max_cudagraph_capture_size,
|
||||
)
|
||||
try:
|
||||
self.dcp_world_size = get_dcp_group().world_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group
|
||||
self.dcp_kv_cache_interleave_size = (
|
||||
vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
)
|
||||
except AssertionError:
|
||||
# DCP might not be initialized in testing
|
||||
self.dcp_world_size = 1
|
||||
self.dcp_rank = 0
|
||||
self.dcp_kv_cache_interleave_size = 1
|
||||
self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank(
|
||||
kv_cache_spec.dcp_sharded
|
||||
)
|
||||
self.dcp_kv_cache_interleave_size = (
|
||||
vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
)
|
||||
self.use_dcp = self.dcp_world_size > 1
|
||||
self.dcp_a2a = (
|
||||
self.use_dcp and vllm_config.parallel_config.dcp_comm_backend == "a2a"
|
||||
@@ -1424,6 +1423,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
# Leave prefill/decode/cascade_wrapper empty, to be populated
|
||||
# case by case depending on the batch contents and backend selection.
|
||||
attn_metadata = FlashInferMetadata(
|
||||
kv_cache_layout=self.kv_cache_layout,
|
||||
num_actual_tokens=num_actual_tokens,
|
||||
slot_mapping=common_attn_metadata.slot_mapping,
|
||||
q_data_type_prefill=self.q_data_type_prefill,
|
||||
@@ -1904,8 +1904,6 @@ class FlashInferImpl(AttentionImpl):
|
||||
num_heads, num_kv_heads, is_prefill=False
|
||||
)
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
# The layout is resolved after model construction, so read it lazily.
|
||||
self.cache_config = vllm_config.cache_config if vllm_config else None
|
||||
# Query pre-quantization needs a single dtype for the whole query tensor.
|
||||
# SM90 XQA needs BF16/FP16-Q for decode and FP8 for prefill,
|
||||
# so only enable this for SM100 trtllm-gen where both use FP8-Q.
|
||||
@@ -1941,11 +1939,6 @@ class FlashInferImpl(AttentionImpl):
|
||||
else:
|
||||
self.dcp_combine = partial(cp_lse_ag_out_rs, is_lse_base_on_e=False)
|
||||
|
||||
@property
|
||||
def kv_cache_layout(self) -> KVCacheLayout:
|
||||
assert self.cache_config is not None
|
||||
return self.cache_config.get_resolved_kv_cache_layout()
|
||||
|
||||
def fused_output_quant_supported(self, quant_key: QuantKey):
|
||||
if quant_key == kNvfp4Dynamic and self.is_kvcache_nvfp4:
|
||||
logger.warning_once(
|
||||
@@ -2111,6 +2104,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
# performance to make sure it does not introduce any overhead.
|
||||
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
kv_cache_layout = attn_metadata.kv_cache_layout
|
||||
|
||||
# FlashInfer treats uint8 KV cache as NVFP4. vLLM stores FP8 KV cache
|
||||
# as uint8 bytes, so pass FP8 caches with their logical dtype.
|
||||
@@ -2134,7 +2128,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
if attn_metadata.use_cascade:
|
||||
# Cascade attention (rare case).
|
||||
assert attn_metadata.cascade_wrapper is not None
|
||||
stride_order = self.kv_cache_layout.layer_view_order
|
||||
stride_order = kv_cache_layout.layer_view_order
|
||||
if self.is_kvcache_nvfp4:
|
||||
kv_cache_views = tuple(
|
||||
cache.permute(*stride_order)
|
||||
@@ -2154,7 +2148,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
num_prefill_tokens = attn_metadata.num_prefill_tokens
|
||||
|
||||
stride_order = self.kv_cache_layout.layer_view_order
|
||||
stride_order = kv_cache_layout.layer_view_order
|
||||
kv_cache_permute = kv_cache.permute(*stride_order) # HND and contiguous
|
||||
# Fix degenerate strides on any size-1 dimension (e.g. num_kv_heads=1
|
||||
# with TP=8). PyTorch permits non-canonical strides on size-1 dims;
|
||||
@@ -2311,7 +2305,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
seq_lens_prefill = attn_metadata.prefill.seq_lens
|
||||
|
||||
# This path needs to be enabled with VLLM_KV_CACHE_LAYOUT = HND
|
||||
assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND"
|
||||
assert get_flashinfer_layout_string(kv_cache_layout) == "HND"
|
||||
assert is_strictly_contiguous(prefill_query)
|
||||
assert is_strictly_contiguous(workspace_buffer)
|
||||
assert is_strictly_contiguous(block_tables_prefill)
|
||||
@@ -2501,7 +2495,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
# trtllm-gen needs HND layout on SM100. XQA is selected
|
||||
# separately on SM90 and does not use this SM100 layout gate.
|
||||
if decode_with_trtllm_gen:
|
||||
assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND"
|
||||
assert get_flashinfer_layout_string(kv_cache_layout) == "HND"
|
||||
else:
|
||||
assert decode_with_xqa
|
||||
assert is_strictly_contiguous(decode_query)
|
||||
@@ -2547,7 +2541,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
window_left=self.window_left,
|
||||
out=output_padded[:decode_query_tokens],
|
||||
sinks=self.sinks,
|
||||
kv_layout=get_flashinfer_layout_string(self.kv_cache_layout),
|
||||
kv_layout=get_flashinfer_layout_string(kv_cache_layout),
|
||||
q_len_per_req=q_len_per_req,
|
||||
mask=attn_metadata.decode.mask,
|
||||
q_cu_seq_lens=attn_metadata.decode.q_cu_seq_lens,
|
||||
@@ -2614,7 +2608,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
sinks=self.sinks,
|
||||
o_sf_scale=self.o_sf_scale,
|
||||
out=out,
|
||||
kv_layout=get_flashinfer_layout_string(self.kv_cache_layout),
|
||||
kv_layout=get_flashinfer_layout_string(kv_cache_layout),
|
||||
backend=attn_metadata.decode.kernel.value,
|
||||
q_len_per_req=q_len_per_req,
|
||||
max_q_len=max_q_len,
|
||||
|
||||
@@ -21,7 +21,12 @@ from vllm.config.cache import _layout_from_name
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor
|
||||
from vllm.v1.kv_cache_interface import KVCacheLayout, KVCacheSpec, MambaSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
@@ -286,9 +291,10 @@ def resolve_kv_cache_layout(
|
||||
# specs can re-interpret HNC with different sizes as long as the total number of
|
||||
# bytes is the same. If not block-compact, each spec must agree on HNC to alias
|
||||
# the same page (this aliasing is done by the Hybrid Memory Allocator, HMA).
|
||||
kv_cache_specs = tuple(kv_cache_specs or ())
|
||||
hnc_shapes = {
|
||||
(spec.num_heads, spec.num_states, spec.page_size_bytes)
|
||||
for spec in kv_cache_specs or ()
|
||||
for spec in kv_cache_specs
|
||||
}
|
||||
if len(hnc_shapes) > 1:
|
||||
candidates = [m for m in candidates if m.is_block_compact]
|
||||
@@ -298,6 +304,20 @@ def resolve_kv_cache_layout(
|
||||
f"none is in every supported set: {supported_layouts}."
|
||||
)
|
||||
|
||||
dcp_sharding = {
|
||||
spec.dcp_sharded for spec in kv_cache_specs if isinstance(spec, AttentionSpec)
|
||||
}
|
||||
page_sizes = {spec.page_size_bytes for spec in kv_cache_specs}
|
||||
if len(dcp_sharding) > 1 and len(page_sizes) > 1:
|
||||
# Sharded target and replicated draft caches need independent groups.
|
||||
# Block-outer layouts can pack those groups without equalizing pages.
|
||||
candidates = [layout for layout in candidates if layout.is_block_outermost]
|
||||
if not candidates:
|
||||
raise ValueError(
|
||||
"DCP with a replicated draft and mixed KV page sizes requires "
|
||||
f"a block-outer KV cache layout; supported sets: {supported_layouts}."
|
||||
)
|
||||
|
||||
if (requested := envs.VLLM_KV_CACHE_LAYOUT) is not None:
|
||||
layout = _layout_from_name(requested)
|
||||
if layout not in candidates:
|
||||
|
||||
@@ -11,7 +11,6 @@ from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
KVCacheBlock,
|
||||
dcp_world_size_for_kv_cache_spec,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
CrossAttentionManager,
|
||||
@@ -20,6 +19,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
get_manager_for_kv_cache_spec,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheSpec,
|
||||
@@ -140,9 +140,7 @@ class KVCacheCoordinator(ABC):
|
||||
role=kv_cache_group.role,
|
||||
enable_caching=enable_caching,
|
||||
kv_cache_group_id=i,
|
||||
dcp_world_size=dcp_world_size_for_kv_cache_spec(
|
||||
kv_cache_group.kv_cache_spec, dcp_world_size
|
||||
),
|
||||
dcp_world_size=dcp_world_size,
|
||||
pcp_world_size=pcp_world_size,
|
||||
scheduler_block_size=self.scheduler_block_size,
|
||||
needs_kv_cache_zeroing=self.kv_cache_config.needs_kv_cache_zeroing,
|
||||
@@ -664,16 +662,18 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
f"hash_block_size. block_sizes={cacheable_block_sizes}, "
|
||||
f"hash_block_size={hash_block_size}"
|
||||
)
|
||||
assert pcp_world_size == 1, "PCP not support hybrid attn now."
|
||||
if dcp_world_size > 1:
|
||||
# DCP shards full-attention KV across ranks and replicates Mamba
|
||||
# state; other spec types (e.g. sliding window) have no DCP-aware
|
||||
# handling yet, so reject them explicitly.
|
||||
for g in kv_cache_config.kv_cache_groups:
|
||||
assert isinstance(g.kv_cache_spec, (FullAttentionSpec, MambaSpec)), (
|
||||
for g in kv_cache_config.kv_cache_groups:
|
||||
spec = g.kv_cache_spec
|
||||
replicated = not spec.dcp_sharded
|
||||
if pcp_world_size > 1:
|
||||
assert isinstance(spec, FullAttentionSpec) or (
|
||||
isinstance(spec, AttentionSpec) and replicated
|
||||
), "PCP only supports full attention and replicated draft groups."
|
||||
if dcp_world_size > 1:
|
||||
assert isinstance(spec, FullAttentionSpec) or replicated, (
|
||||
"DCP with hybrid KV cache layouts only supports "
|
||||
"full-attention and Mamba groups, got: "
|
||||
f"{type(g.kv_cache_spec).__name__}."
|
||||
"full-attention, Mamba, and replicated draft groups, got: "
|
||||
f"{type(spec).__name__}."
|
||||
)
|
||||
# Fine-grained hash hits require Mamba "align" and compatible cache
|
||||
# managers in every group. TP needs hashing finer than the Mamba block;
|
||||
|
||||
@@ -682,12 +682,7 @@ def hash_block_tokens(
|
||||
|
||||
def resolve_dcp_kv_block_size(spec: KVCacheSpec, dcp_world_size: int) -> int:
|
||||
"""Return the token span of a cache block under DCP."""
|
||||
layer_specs = iter_layer_specs(spec)
|
||||
if len(layer_specs) > 0 and all(
|
||||
isinstance(layer_spec, AttentionSpec) for layer_spec in layer_specs
|
||||
):
|
||||
return spec.block_size * dcp_world_size
|
||||
return spec.block_size
|
||||
return spec.block_size * (dcp_world_size if spec.dcp_sharded else 1)
|
||||
|
||||
|
||||
def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCacheSpec:
|
||||
@@ -707,28 +702,6 @@ def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCache
|
||||
return replace(spec, block_size=block_size)
|
||||
|
||||
|
||||
def dcp_world_size_for_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> int:
|
||||
"""Return the DCP size that owns this group's block geometry.
|
||||
|
||||
Full-attention KV (including MLA) is sharded across DCP ranks, so prefix
|
||||
hashing and manager ``block_size`` use the process DCP size. Other specs
|
||||
keep replicated per-rank state (Mamba, sliding window, chunked-local) and
|
||||
must keep ``dcp_world_size=1`` even when the process runs with DCP > 1.
|
||||
|
||||
Draft MLA groups on the sharded DSpark path are ``FullAttentionSpec`` /
|
||||
``MLAAttentionSpec`` and therefore keep the process DCP size. A replicated
|
||||
draft group would need a different spec, not this helper.
|
||||
"""
|
||||
if dcp_world_size <= 1:
|
||||
return 1
|
||||
inner = spec
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
inner = next(iter(spec.kv_cache_specs.values()))
|
||||
if isinstance(inner, FullAttentionSpec):
|
||||
return dcp_world_size
|
||||
return 1
|
||||
|
||||
|
||||
def resolve_kv_cache_block_sizes(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -752,6 +725,8 @@ def resolve_kv_cache_block_sizes(
|
||||
groups = kv_cache_config.kv_cache_groups
|
||||
|
||||
if len(groups) <= 1:
|
||||
if groups and not groups[0].kv_cache_spec.dcp_sharded:
|
||||
dcp = 1
|
||||
bs = cache_config.block_size * dcp
|
||||
return bs, bs
|
||||
|
||||
@@ -1935,18 +1910,21 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]):
|
||||
kv_cache_spec: The kv cache spec of each attention layer in the model
|
||||
|
||||
"""
|
||||
if is_kv_cache_spec_uniform(
|
||||
kv_cache_spec
|
||||
) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec):
|
||||
return
|
||||
|
||||
logger.warning(
|
||||
"Hybrid KV cache manager is disabled for this hybrid model, "
|
||||
"This means we do not enable any optimizations for saving KV cache "
|
||||
"memory (e.g., dropping the KV cache outside the sliding window). "
|
||||
"The compute of layers like sliding window is still saved."
|
||||
)
|
||||
kv_cache_spec.update(_promote_local_kv_cache_specs(kv_cache_spec))
|
||||
groups: defaultdict[bool, dict[str, KVCacheSpec]] = defaultdict(dict)
|
||||
for name, spec in kv_cache_spec.items():
|
||||
replicated = isinstance(spec, AttentionSpec) and not spec.dcp_sharded
|
||||
groups[replicated][name] = spec
|
||||
for specs in groups.values():
|
||||
promoted_specs = _promote_local_kv_cache_specs(specs)
|
||||
if promoted_specs == specs:
|
||||
continue
|
||||
logger.warning(
|
||||
"Hybrid KV cache manager is disabled for this hybrid model, "
|
||||
"This means we do not enable any optimizations for saving KV cache "
|
||||
"memory (e.g., dropping the KV cache outside the sliding window). "
|
||||
"The compute of layers like sliding window is still saved."
|
||||
)
|
||||
kv_cache_spec.update(promoted_specs)
|
||||
|
||||
|
||||
def _approximate_gcd(values: Sequence[int], *, lower_bound: int | None = None) -> int:
|
||||
@@ -2276,7 +2254,7 @@ def _ensure_min_page_size(
|
||||
scaled: list[KVCacheGroupSpec] = []
|
||||
for g in groups:
|
||||
s = g.kv_cache_spec
|
||||
kw: dict[str, int] = {"block_size": s.block_size * scale}
|
||||
kw: dict[str, Any] = {"block_size": s.block_size * scale}
|
||||
if isinstance(s, (AttentionSpec, MambaSpec)) and s.page_size_padded is not None:
|
||||
kw["page_size_padded"] = s.page_size_padded * scale
|
||||
scaled.append(KVCacheGroupSpec(g.layer_names, replace(s, **kw)))
|
||||
|
||||
@@ -106,11 +106,9 @@ class SingleTypeKVCacheManager(ABC):
|
||||
# managers have been validated by the coordinator.
|
||||
self.cache_hit_alignment_tokens = scheduler_block_size
|
||||
# The block size for this manager; used for actual block allocation.
|
||||
self.block_size = kv_cache_spec.block_size
|
||||
self.dcp_world_size = dcp_world_size
|
||||
self.pcp_world_size = pcp_world_size
|
||||
if dcp_world_size > 1:
|
||||
self.block_size *= dcp_world_size
|
||||
self.dcp_world_size = dcp_world_size if kv_cache_spec.dcp_sharded else 1
|
||||
self.pcp_world_size = pcp_world_size if kv_cache_spec.dcp_sharded else 1
|
||||
self.block_size = kv_cache_spec.block_size * self.dcp_world_size
|
||||
self.kv_cache_spec = kv_cache_spec
|
||||
self.block_pool = block_pool
|
||||
self.enable_caching = enable_caching
|
||||
@@ -1455,10 +1453,6 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, block_pool, **kwargs)
|
||||
# Mamba layers use TP instead of DCP, so each rank holds the full
|
||||
# recurrent state. Undo the DCP/PCP block_size scaling that the base
|
||||
# class applies for attention groups whose KV cache is partitioned.
|
||||
self.block_size = kv_cache_spec.block_size
|
||||
self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode
|
||||
self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks
|
||||
self.has_prefill_checkpoint_blocks = (
|
||||
|
||||
@@ -158,6 +158,8 @@ class KVCacheSpec:
|
||||
|
||||
# number of tokens in a block
|
||||
block_size: int
|
||||
dcp_sharded: bool = field(default=False, kw_only=True)
|
||||
"""Whether DCP shards this cache's token positions across ranks."""
|
||||
|
||||
block_stride_alignment: int | None = field(default=None, kw_only=True)
|
||||
"""Required byte alignment between physical blocks, including packed layers."""
|
||||
@@ -481,6 +483,7 @@ class HiSparseResidentSpec(KVCacheSpec):
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class AttentionSpec(KVCacheSpec):
|
||||
dcp_sharded: bool = True
|
||||
num_kv_heads: int
|
||||
head_size: int
|
||||
dtype: torch.dtype
|
||||
@@ -536,7 +539,9 @@ class AttentionSpec(KVCacheSpec):
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
kv_shard_count = parallel_config.decode_context_parallel_size
|
||||
kv_shard_count = (
|
||||
parallel_config.decode_context_parallel_size if self.dcp_sharded else 1
|
||||
)
|
||||
return cdiv(max_len, self.block_size * kv_shard_count)
|
||||
|
||||
|
||||
@@ -568,7 +573,7 @@ class FullAttentionSpec(AttentionSpec):
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size > 1:
|
||||
if self.dcp_sharded and dcp_world_size > 1:
|
||||
max_model_len = cdiv(max_model_len, dcp_world_size)
|
||||
return cdiv(max_model_len, self.block_size) * self.page_size_bytes
|
||||
|
||||
@@ -612,6 +617,7 @@ class FullAttentionSpec(AttentionSpec):
|
||||
head_size_v=specs[0].head_size_v,
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -698,6 +704,7 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -760,6 +767,7 @@ class RSWASpec(FullAttentionSpec):
|
||||
head_size_v=base.head_size_v,
|
||||
dtype=base.dtype,
|
||||
kv_quant_mode=base.kv_quant_mode,
|
||||
dcp_sharded=base.dcp_sharded,
|
||||
page_size_padded=base.page_size_padded,
|
||||
num_head_slots=base.num_head_slots,
|
||||
state_content_bytes=base.state_content_bytes,
|
||||
@@ -850,9 +858,10 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
return cdiv(num_tokens, self.block_size) + 1
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
assert vllm_config.parallel_config.decode_context_parallel_size == 1, (
|
||||
"DCP not support sliding window."
|
||||
)
|
||||
assert (
|
||||
not self.dcp_sharded
|
||||
or vllm_config.parallel_config.decode_context_parallel_size == 1
|
||||
), "DCP not support sliding window."
|
||||
max_blocks = self.max_admission_blocks_per_request(
|
||||
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
||||
max_model_len=vllm_config.model_config.max_model_len,
|
||||
@@ -950,6 +959,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
|
||||
bounded_replay_set = set(spec.bounded_replay for spec in specs)
|
||||
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
|
||||
assert len({spec.dcp_sharded for spec in specs}) == 1
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
@@ -969,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
num_kv_heads=specs[0].num_kv_heads,
|
||||
head_size=specs[0].head_size,
|
||||
dtype=specs[0].dtype,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -1185,6 +1196,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
block_stride_alignment=specs[0].block_stride_alignment,
|
||||
dtype=specs[0].dtype,
|
||||
kv_quant_mode=specs[0].kv_quant_mode,
|
||||
dcp_sharded=specs[0].dcp_sharded,
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
num_head_slots=specs[0].num_head_slots,
|
||||
state_content_bytes=specs[0].state_content_bytes,
|
||||
@@ -1217,6 +1229,11 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
|
||||
kv_cache_specs: dict[str, KVCacheSpec]
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
if self.kv_cache_specs:
|
||||
object.__setattr__(self, "dcp_sharded", self.first_spec.dcp_sharded)
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values())
|
||||
@@ -1270,6 +1287,8 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
if len(block_sizes) > 1:
|
||||
# Different block sizes, not uniform.
|
||||
return False
|
||||
if len({spec.dcp_sharded for spec in kv_cache_specs.values()}) > 1:
|
||||
return False
|
||||
first_spec = next(iter(kv_cache_specs.values()))
|
||||
return first_spec.is_uniform_with_collection(kv_cache_specs)
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import contextlib
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field, replace
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
@@ -27,13 +27,13 @@ from vllm.v1.core.kv_cache_coordinator import (
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHashWithGroupId,
|
||||
ExternalBlockHash,
|
||||
dcp_world_size_for_kv_cache_spec,
|
||||
generate_block_hash_extra_keys,
|
||||
get_block_hash,
|
||||
get_group_id,
|
||||
make_block_hash_with_group_id,
|
||||
maybe_convert_block_hash,
|
||||
resolve_block_hashes,
|
||||
resolve_dcp_kv_block_size,
|
||||
)
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -48,6 +48,11 @@ from vllm.v1.simple_kv_offload.metadata import (
|
||||
SimpleCPUOffloadMetadata,
|
||||
SimpleCPUOffloadWorkerMetadata,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
OUTCOME_TO_FIELD,
|
||||
MetricName,
|
||||
SimpleCPUOffloadStats,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
@@ -118,7 +123,8 @@ class BoundaryStoreStats:
|
||||
symptom is a lower cache hit rate with nothing in the logs, so each decline
|
||||
reason is counted and exposed through
|
||||
``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by
|
||||
``reset()``; not otherwise cleared.
|
||||
``reset()``, which carries any not-yet-drained deltas; not otherwise
|
||||
cleared.
|
||||
"""
|
||||
|
||||
published: int = 0
|
||||
@@ -143,6 +149,7 @@ class SimpleCPUOffloadScheduler:
|
||||
hash_block_size: int,
|
||||
lazy_offload: bool = False,
|
||||
disk_capacity_bytes: int = 0,
|
||||
use_page_cache: bool = False,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
@@ -169,6 +176,13 @@ class SimpleCPUOffloadScheduler:
|
||||
self.cpu_kv_cache_config.prefix_cacheable_group_ids
|
||||
)
|
||||
self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU
|
||||
# Must track kv_event_medium.
|
||||
self._info_labelvalues: tuple[str, ...] = (
|
||||
"disk" if disk_capacity_bytes > 0 else "cpu",
|
||||
str(use_page_cache).lower(),
|
||||
str(lazy_offload).lower(),
|
||||
str(self.num_cpu_blocks),
|
||||
)
|
||||
# Find the full attention kv group for prefix cache matching.
|
||||
self.fa_gidx = -1
|
||||
for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups):
|
||||
@@ -251,6 +265,9 @@ class SimpleCPUOffloadScheduler:
|
||||
self._load_event_counter: int = 0
|
||||
self._store_event_counter: int = 0
|
||||
self.boundary_store_stats = BoundaryStoreStats()
|
||||
# Interval stats state drained by get_stats()
|
||||
self._boundary_stats_snapshot = BoundaryStoreStats()
|
||||
self._interval_load_blocks_completed = 0
|
||||
|
||||
# For TP/PP: track partial store completions across steps.
|
||||
# Events must be reported by all world_size workers before considered complete.
|
||||
@@ -302,11 +319,7 @@ class SimpleCPUOffloadScheduler:
|
||||
target = 0
|
||||
for g in kv_cache_config.prefix_cacheable_groups:
|
||||
spec = g.kv_cache_spec
|
||||
# Only full attention is sharded across DCP ranks; replicated specs
|
||||
# (mamba, sliding window, chunked-local) keep their own block size.
|
||||
block_size = spec.block_size * dcp_world_size_for_kv_cache_spec(
|
||||
spec, cp_world_size
|
||||
)
|
||||
block_size = resolve_dcp_kv_block_size(spec, cp_world_size)
|
||||
if isinstance(spec, MambaSpec):
|
||||
target += 2
|
||||
elif isinstance(spec, SlidingWindowSpec):
|
||||
@@ -955,6 +968,42 @@ class SimpleCPUOffloadScheduler:
|
||||
def get_boundary_store_stats(self) -> BoundaryStoreStats:
|
||||
return replace(self.boundary_store_stats)
|
||||
|
||||
def get_stats(self) -> SimpleCPUOffloadStats:
|
||||
"""Drain per-step stats for the connector's stats hooks."""
|
||||
stats = SimpleCPUOffloadStats()
|
||||
|
||||
current = self.boundary_store_stats
|
||||
for outcome, field_name in OUTCOME_TO_FIELD.items():
|
||||
delta = getattr(current, field_name) - getattr(
|
||||
self._boundary_stats_snapshot, field_name
|
||||
)
|
||||
if delta > 0:
|
||||
stats.increase_counter(MetricName.SAVE_OUTCOMES, delta, (outcome,))
|
||||
self._boundary_stats_snapshot = replace(current)
|
||||
|
||||
if self._interval_load_blocks_completed:
|
||||
stats.increase_counter(
|
||||
MetricName.LOAD_BLOCKS,
|
||||
self._interval_load_blocks_completed,
|
||||
)
|
||||
self._interval_load_blocks_completed = 0
|
||||
|
||||
stats.set_gauge(
|
||||
MetricName.USED_BLOCKS,
|
||||
self.num_cpu_blocks - self.cpu_block_pool.get_num_free_blocks(),
|
||||
)
|
||||
pending = sum(
|
||||
len(t.cpu_block_ids)
|
||||
for t in (
|
||||
*self._store_event_to_blocks.values(),
|
||||
*self._pending_finished_stores,
|
||||
*self._abandoned_store_event_to_blocks.values(),
|
||||
)
|
||||
)
|
||||
stats.set_gauge(MetricName.PENDING_STORE_BLOCKS, pending)
|
||||
stats.set_gauge(MetricName.INFO, 1, self._info_labelvalues)
|
||||
return stats
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Handle async transfer completions from worker.
|
||||
|
||||
@@ -965,7 +1014,9 @@ class SimpleCPUOffloadScheduler:
|
||||
"""
|
||||
# --- Load completions ---
|
||||
for req_id in list(connector_output.finished_recving or []):
|
||||
self._cleanup_load_request(req_id)
|
||||
completed_blocks = self._cleanup_load_request(req_id)
|
||||
if completed_blocks:
|
||||
self._interval_load_blocks_completed += completed_blocks
|
||||
|
||||
# --- Store completions ---
|
||||
meta = connector_output.kv_connector_worker_meta
|
||||
@@ -1270,18 +1321,22 @@ class SimpleCPUOffloadScheduler:
|
||||
if blocks_to_free:
|
||||
self.cpu_block_pool.free_blocks(blocks_to_free)
|
||||
|
||||
def _cleanup_load_request(self, req_id: str) -> None:
|
||||
def _cleanup_load_request(self, req_id: str) -> int:
|
||||
"""Release all load resources for a request.
|
||||
|
||||
Shared between request_finished() and update_connector_output() paths.
|
||||
Removes the request from _reqs_to_load, cleans up event mappings,
|
||||
and frees CPU/GPU touch refs.
|
||||
|
||||
Returns the number of blocks in the load if it had been issued to
|
||||
the worker, else 0 (never-issued loads are not counted as completed
|
||||
by the caller).
|
||||
"""
|
||||
state = self._reqs_to_load.pop(req_id, None)
|
||||
if state is None:
|
||||
state = self._abandoned_reqs_to_load.pop(req_id, None)
|
||||
if state is None:
|
||||
return
|
||||
return 0
|
||||
# Remove from load event mapping (only this req, not whole event)
|
||||
if state.load_event is not None:
|
||||
reqs = self._load_event_to_reqs.get(state.load_event)
|
||||
@@ -1304,6 +1359,10 @@ class SimpleCPUOffloadScheduler:
|
||||
for bid in state.transfer_meta.gpu_block_ids
|
||||
)
|
||||
|
||||
if state.load_event is None:
|
||||
return 0
|
||||
return len(state.transfer_meta.gpu_block_ids)
|
||||
|
||||
def _cleanup_store_request(self, req_id: str) -> None:
|
||||
"""Release store metadata for a request.
|
||||
|
||||
@@ -1363,7 +1422,20 @@ class SimpleCPUOffloadScheduler:
|
||||
if event_idx in self._abandoned_store_event_to_blocks
|
||||
}
|
||||
self._cursor = None
|
||||
self.boundary_store_stats = BoundaryStoreStats()
|
||||
# Seed the fresh counters with outcomes since the last drain so a
|
||||
# mid-interval reset() does not drop them from the next report.
|
||||
undrained = BoundaryStoreStats(
|
||||
**{
|
||||
f.name: max(
|
||||
0,
|
||||
getattr(self.boundary_store_stats, f.name)
|
||||
- getattr(self._boundary_stats_snapshot, f.name),
|
||||
)
|
||||
for f in fields(BoundaryStoreStats)
|
||||
}
|
||||
)
|
||||
self.boundary_store_stats = undrained
|
||||
self._boundary_stats_snapshot = BoundaryStoreStats()
|
||||
# NOTE: _load_event_counter / _store_event_counter are not
|
||||
# reset as they are monotonic and must stay ahead of the workers
|
||||
# high-water marks to avoid event index collisions
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Prometheus metrics for SimpleCPUOffloadConnector.
|
||||
|
||||
The stats payload uses the same self-describing wire format as the
|
||||
offloading connector, so it survives IPC serialization without the
|
||||
metric classes on the receiving side.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
|
||||
|
||||
class MetricName:
|
||||
"""Metric names for SimpleCPUOffloadConnector."""
|
||||
|
||||
SAVE_OUTCOMES = "vllm:simple_kv_offload_save_outcomes_total"
|
||||
LOAD_BLOCKS = "vllm:simple_kv_offload_load_blocks_total"
|
||||
USED_BLOCKS = "vllm:simple_kv_offload_used_blocks"
|
||||
PENDING_STORE_BLOCKS = "vllm:simple_kv_offload_pending_store_blocks"
|
||||
INFO = "vllm:simple_kv_offload_info"
|
||||
|
||||
|
||||
# outcome label value -> BoundaryStoreStats field name.
|
||||
# ``published`` is the sum of all outcomes and is not labeled.
|
||||
OUTCOME_TO_FIELD = {
|
||||
"stored": "stored",
|
||||
"dropped_cpu_full": "dropped_cpu_full",
|
||||
"dropped_request_gone": "dropped_request_gone",
|
||||
"dropped_null_block": "dropped_null_block",
|
||||
"dropped_not_hashed": "dropped_not_hashed",
|
||||
"skipped_already_cached": "skipped_already_cached",
|
||||
"skipped_in_flight": "skipped_in_flight",
|
||||
}
|
||||
|
||||
INFO_LABELS = ["backend", "page_cache", "lazy_offload", "capacity_blocks"]
|
||||
|
||||
|
||||
class _MetricType:
|
||||
"""Type tags embedded in the serialized stats payload."""
|
||||
|
||||
COUNTER = "counter"
|
||||
GAUGE = "gauge"
|
||||
|
||||
|
||||
class _StatsKey:
|
||||
"""Top-level keys in the serialized stats dict."""
|
||||
|
||||
# Maps metric name -> _MetricType value
|
||||
TYPES = "types"
|
||||
# Maps metric name -> {label values tuple -> observed value}
|
||||
DATA = "data"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SimpleCPUOffloadStats(KVConnectorStats):
|
||||
"""Stats payload for SimpleCPUOffloadConnector.
|
||||
|
||||
The ``data`` dict is structured as::
|
||||
|
||||
{
|
||||
_StatsKey.TYPES: {name: _MetricType.*, ...},
|
||||
_StatsKey.DATA: {name: {labelvalues: value, ...}, ...},
|
||||
}
|
||||
|
||||
Counter values are summed when aggregating, gauge values keep the
|
||||
latest snapshot, and unlabeled metrics use ``()`` as their
|
||||
labelvalues tuple.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
if _StatsKey.DATA not in self.data:
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.data: dict[str, Any] = {
|
||||
_StatsKey.TYPES: {},
|
||||
_StatsKey.DATA: {},
|
||||
}
|
||||
|
||||
@property
|
||||
def _types(self) -> dict[str, str]:
|
||||
return self.data[_StatsKey.TYPES]
|
||||
|
||||
@property
|
||||
def _values(self) -> dict[str, Any]:
|
||||
return self.data[_StatsKey.DATA]
|
||||
|
||||
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
|
||||
if other.is_empty():
|
||||
return self
|
||||
assert isinstance(other, SimpleCPUOffloadStats)
|
||||
for key, other_label_values in other._values.items():
|
||||
type_str = other._types.get(key)
|
||||
if type_str is None:
|
||||
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
|
||||
self._types.setdefault(key, type_str)
|
||||
current_label_values = self._values.setdefault(key, {})
|
||||
for labelvalues, value in other_label_values.items():
|
||||
if type_str == _MetricType.COUNTER:
|
||||
assert isinstance(value, int | float)
|
||||
current_label_values[labelvalues] = (
|
||||
current_label_values.get(labelvalues, 0) + value
|
||||
)
|
||||
elif type_str == _MetricType.GAUGE:
|
||||
assert isinstance(value, int | float)
|
||||
current_label_values[labelvalues] = value
|
||||
else:
|
||||
raise AssertionError(
|
||||
f"Unknown metric type '{type_str}' for key: {key}"
|
||||
)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
"""Reduce the interval observations to a flat summary for logging."""
|
||||
return_dict: dict[str, int | float] = {}
|
||||
for key, label_value_map in self._values.items():
|
||||
if key == MetricName.INFO:
|
||||
# Constant config fingerprint; exclude from the log summary.
|
||||
continue
|
||||
type_str = self._types.get(key)
|
||||
if type_str is None:
|
||||
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
|
||||
for labelvalues, value in label_value_map.items():
|
||||
key_with_labels = f"{key}:{labelvalues}" if labelvalues else key
|
||||
if type_str in (_MetricType.COUNTER, _MetricType.GAUGE):
|
||||
assert isinstance(value, int | float)
|
||||
return_dict[key_with_labels] = value
|
||||
else:
|
||||
raise AssertionError(
|
||||
f"Unknown metric type '{type_str}' for key: {key}"
|
||||
)
|
||||
return return_dict
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data.get(_StatsKey.DATA)
|
||||
|
||||
def increase_counter(
|
||||
self,
|
||||
counter_name: str,
|
||||
counter_increase_value: int | float = 1,
|
||||
labelvalues: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
"""Increase a counter on the stats payload."""
|
||||
self._types.setdefault(counter_name, _MetricType.COUNTER)
|
||||
counter_values = self._values.setdefault(counter_name, {})
|
||||
counter_values[labelvalues] = (
|
||||
counter_values.get(labelvalues, 0) + counter_increase_value
|
||||
)
|
||||
|
||||
def set_gauge(
|
||||
self,
|
||||
gauge_name: str,
|
||||
gauge_value: int | float,
|
||||
labelvalues: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
"""Set a gauge snapshot on the stats payload."""
|
||||
self._types.setdefault(gauge_name, _MetricType.GAUGE)
|
||||
gauge_values = self._values.setdefault(gauge_name, {})
|
||||
gauge_values[labelvalues] = gauge_value
|
||||
|
||||
|
||||
class SimpleCPUOffloadPromMetrics(KVConnectorPromMetrics):
|
||||
"""Prometheus registration and recording for SimpleCPUOffloadConnector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
# (engine_idx, metric_name, labelvalues) -> bound child metric
|
||||
self._metrics: dict[tuple[int, str, tuple[str, ...]], PromMetricT] = {}
|
||||
|
||||
self._counter_save_outcomes = self._counter_cls(
|
||||
name="vllm:simple_kv_offload_save_outcomes_total",
|
||||
documentation=(
|
||||
"Store admission decisions in SimpleCPUOffloadConnector, by "
|
||||
"outcome. Outcomes classify eager-mode boundary hand-off "
|
||||
"stores; lazy-mode stores are not classified."
|
||||
),
|
||||
labelnames=self._labelnames + ["outcome"],
|
||||
)
|
||||
self._counter_load_blocks = self._counter_cls(
|
||||
name="vllm:simple_kv_offload_load_blocks_total",
|
||||
documentation=(
|
||||
"KV blocks restored to GPU from the offload pool, counted "
|
||||
"when a load finishes. This is prefill work avoided "
|
||||
"through offload cache hits."
|
||||
),
|
||||
labelnames=self._labelnames,
|
||||
)
|
||||
self._gauge_used_blocks = self._gauge_cls(
|
||||
name="vllm:simple_kv_offload_used_blocks",
|
||||
documentation=(
|
||||
"Offload-pool blocks currently pinned by in-flight "
|
||||
"transfers or cache hits (capacity minus free). Evictable "
|
||||
"cached blocks are not counted; capacity_blocks is on "
|
||||
"vllm:simple_kv_offload_info."
|
||||
),
|
||||
labelnames=self._labelnames,
|
||||
multiprocess_mode="mostrecent",
|
||||
)
|
||||
self._gauge_pending_store_blocks = self._gauge_cls(
|
||||
name="vllm:simple_kv_offload_pending_store_blocks",
|
||||
documentation=(
|
||||
"Offload blocks in pending stores: queued for dispatch, "
|
||||
"issued to workers, or awaiting release after a cache "
|
||||
"reset. A persistently growing value indicates a stuck "
|
||||
"transfer."
|
||||
),
|
||||
labelnames=self._labelnames,
|
||||
multiprocess_mode="mostrecent",
|
||||
)
|
||||
self._gauge_info = self._gauge_cls(
|
||||
name="vllm:simple_kv_offload_info",
|
||||
documentation=(
|
||||
"SimpleCPUOffloadConnector deployment facts. Value is "
|
||||
"always 1. backend is cpu or disk; page_cache and "
|
||||
"lazy_offload are true/false; capacity_blocks is the "
|
||||
"offload pool size."
|
||||
),
|
||||
labelnames=self._labelnames + INFO_LABELS,
|
||||
multiprocess_mode="mostrecent",
|
||||
)
|
||||
|
||||
self._defs: dict[str, PromMetricT] = {
|
||||
MetricName.SAVE_OUTCOMES: self._counter_save_outcomes,
|
||||
MetricName.LOAD_BLOCKS: self._counter_load_blocks,
|
||||
MetricName.USED_BLOCKS: self._gauge_used_blocks,
|
||||
MetricName.PENDING_STORE_BLOCKS: self._gauge_pending_store_blocks,
|
||||
MetricName.INFO: self._gauge_info,
|
||||
}
|
||||
self._num_label_values: dict[str, int] = {
|
||||
MetricName.SAVE_OUTCOMES: 1,
|
||||
MetricName.LOAD_BLOCKS: 0,
|
||||
MetricName.USED_BLOCKS: 0,
|
||||
MetricName.PENDING_STORE_BLOCKS: 0,
|
||||
MetricName.INFO: len(INFO_LABELS),
|
||||
}
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""Observe transfer statistics."""
|
||||
metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
|
||||
metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
|
||||
for key, label_value_map in metric_data.items():
|
||||
prom_metric = self._defs.get(key)
|
||||
if prom_metric is None:
|
||||
raise AssertionError(f"Unknown simple KV offload stats key: {key}")
|
||||
type_str = metric_types.get(key)
|
||||
for labelvalues, value in label_value_map.items():
|
||||
if len(labelvalues) != self._num_label_values[key]:
|
||||
raise AssertionError(
|
||||
f"Metric {key} expects "
|
||||
f"{self._num_label_values[key]} labels, "
|
||||
f"got {len(labelvalues)}"
|
||||
)
|
||||
child = self._metrics.get((engine_idx, key, labelvalues))
|
||||
if child is None:
|
||||
engine_labelvalues = self.per_engine_labelvalues[engine_idx]
|
||||
child = prom_metric.labels(
|
||||
*(engine_labelvalues + list(labelvalues))
|
||||
)
|
||||
self._metrics[(engine_idx, key, labelvalues)] = child
|
||||
if type_str == _MetricType.COUNTER:
|
||||
assert isinstance(value, int | float)
|
||||
child.inc(value)
|
||||
elif type_str == _MetricType.GAUGE:
|
||||
assert isinstance(value, int | float)
|
||||
child.set(value)
|
||||
else:
|
||||
raise AssertionError(
|
||||
f"Unknown metric type '{type_str}' for key: {key}"
|
||||
)
|
||||
@@ -81,13 +81,20 @@ class DraftModelProposer(SpecDecodeBaseProposer):
|
||||
def _create_draft_vllm_config(self) -> VllmConfig:
|
||||
base = super()._create_draft_vllm_config()
|
||||
spec = self.speculative_config
|
||||
target_parallel = base.parallel_config
|
||||
|
||||
return replace(
|
||||
base,
|
||||
quant_config=None,
|
||||
parallel_config=replace(
|
||||
spec.draft_parallel_config,
|
||||
rank=self.vllm_config.parallel_config.rank,
|
||||
rank=target_parallel.rank,
|
||||
decode_context_parallel_size=(
|
||||
target_parallel.decode_context_parallel_size
|
||||
),
|
||||
cp_kv_cache_interleave_size=target_parallel.cp_kv_cache_interleave_size,
|
||||
dcp_comm_backend=target_parallel.dcp_comm_backend,
|
||||
dcp_q_replicate=target_parallel.dcp_q_replicate,
|
||||
),
|
||||
model_config=spec.draft_model_config,
|
||||
)
|
||||
|
||||
@@ -40,6 +40,8 @@ def check_attention_cp_compatibility(
|
||||
layer_impl = getattr(layer, "impl", None)
|
||||
if layer_impl is None:
|
||||
continue
|
||||
if not check_pcp and layer_impl.dcp_world_size == 1:
|
||||
continue
|
||||
if vllm_config.speculative_config is not None and interleave_size > 1:
|
||||
assert layer_impl.supports_mtp_with_cp_non_trivial_interleave_size, (
|
||||
"MTP with cp_kv_cache_interleave_size > 1 is not "
|
||||
|
||||
@@ -27,6 +27,7 @@ class BlockTables:
|
||||
cp_rank: int = 0,
|
||||
cp_interleave: int = 1,
|
||||
slot_mapping_enabled: list[bool] | None = None,
|
||||
dcp_sharded: list[bool] | None = None,
|
||||
):
|
||||
self.block_sizes = block_sizes
|
||||
self.kernel_block_sizes = kernel_block_sizes
|
||||
@@ -44,6 +45,10 @@ class BlockTables:
|
||||
slot_mapping_enabled = [True] * self.num_kv_cache_groups
|
||||
assert len(slot_mapping_enabled) == self.num_kv_cache_groups
|
||||
self._slot_mapping_enabled = slot_mapping_enabled
|
||||
if dcp_sharded is None:
|
||||
dcp_sharded = [True] * self.num_kv_cache_groups
|
||||
assert len(dcp_sharded) == self.num_kv_cache_groups
|
||||
self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device)
|
||||
|
||||
self.blocks_per_kv_block = [
|
||||
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
|
||||
@@ -210,6 +215,7 @@ class BlockTables:
|
||||
self.block_sizes_tensor,
|
||||
self.kernel_block_sizes_tensor,
|
||||
self.slot_mapping_enabled,
|
||||
self.dcp_sharded,
|
||||
slot_mappings,
|
||||
slot_mappings.stride(0),
|
||||
self.cp_rank,
|
||||
@@ -283,6 +289,7 @@ def _compute_slot_mappings_kernel(
|
||||
block_sizes, # [num_kv_cache_groups]
|
||||
kernel_block_sizes, # [num_kv_cache_groups]
|
||||
slot_mapping_enabled, # [num_kv_cache_groups]
|
||||
dcp_sharded, # [num_kv_cache_groups]
|
||||
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
|
||||
slot_mappings_stride,
|
||||
cp_rank,
|
||||
@@ -312,6 +319,8 @@ def _compute_slot_mappings_kernel(
|
||||
kv_block_size = tl.load(block_sizes + group_id)
|
||||
kernel_block_size = tl.load(kernel_block_sizes + group_id)
|
||||
mapping_enabled = tl.load(slot_mapping_enabled + group_id)
|
||||
if CP_SIZE != 1:
|
||||
sharded = tl.load(dcp_sharded + group_id)
|
||||
|
||||
req_state_idx = tl.load(idx_mapping + batch_idx)
|
||||
# idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns
|
||||
@@ -337,6 +346,8 @@ def _compute_slot_mappings_kernel(
|
||||
remainder = virtual_block_offsets % CP_INTERLEAVE
|
||||
local_offsets = rounds * CP_INTERLEAVE + remainder
|
||||
local_positions = virtual_block_indices * kv_block_size + local_offsets
|
||||
local_positions = tl.where(sharded, local_positions, positions)
|
||||
is_local = ~sharded | is_local
|
||||
|
||||
block_indices = tl.where(
|
||||
mapping_enabled, local_positions // kernel_block_size, 0
|
||||
|
||||
@@ -21,6 +21,7 @@ import gc
|
||||
import time
|
||||
from contextlib import AbstractContextManager
|
||||
from copy import deepcopy
|
||||
from dataclasses import replace
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
@@ -561,7 +562,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return encoder_runner.get_encoder_timing_stats()
|
||||
|
||||
def get_kv_cache_spec(self):
|
||||
return get_kv_cache_spec(self.vllm_config)
|
||||
kv_cache_spec = get_kv_cache_spec(self.vllm_config)
|
||||
# A draft running at DCP=1 under a DCP target keeps a replicated cache.
|
||||
if (
|
||||
isinstance(self.speculator, DraftModelSpeculator)
|
||||
and self.speculator.dcp_size == 1
|
||||
and self.dcp_size > 1
|
||||
):
|
||||
for name in self.speculator.draft_attn_layer_names & kv_cache_spec.keys():
|
||||
kv_cache_spec[name] = replace(kv_cache_spec[name], dcp_sharded=False)
|
||||
return kv_cache_spec
|
||||
|
||||
def initialize_kv_cache(
|
||||
self,
|
||||
@@ -587,6 +597,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
block_sizes = []
|
||||
max_num_blocks_per_group = []
|
||||
slot_mapping_enabled = []
|
||||
dcp_sharded = []
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
||||
spec = kv_cache_group.kv_cache_spec
|
||||
block_sizes.append(spec.block_size)
|
||||
@@ -594,6 +605,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec
|
||||
)
|
||||
slot_mapping_enabled.append(layer_spec.uses_slot_mapping)
|
||||
dcp_sharded.append(spec.dcp_sharded)
|
||||
# Let each cache type account for CP. Attention KV is DCP-sharded,
|
||||
# while Mamba/GDN recurrent state is replicated across DCP ranks.
|
||||
max_num_blocks = spec.max_num_blocks_per_req(
|
||||
@@ -657,6 +669,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
device=self.device,
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
slot_mapping_enabled=slot_mapping_enabled,
|
||||
dcp_sharded=dcp_sharded,
|
||||
cp_size=self.dcp_size,
|
||||
cp_rank=self.dcp_rank,
|
||||
cp_interleave=self.cp_interleave,
|
||||
|
||||
@@ -145,7 +145,10 @@ class PCPManager:
|
||||
if speculative_config is not None:
|
||||
if speculative_config.use_dspark():
|
||||
dcp_size = parallel_config.decode_context_parallel_size
|
||||
if dcp_size not in (1, pcp_size):
|
||||
if (
|
||||
dcp_size not in (1, pcp_size)
|
||||
and speculative_config.draft_model_config.use_mla
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"MRV2 PCP DSpark requires DCP=1 or DCP=PCP; got "
|
||||
f"DCP={dcp_size}, PCP={pcp_size}."
|
||||
|
||||
@@ -37,12 +37,18 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if parallel_config.prefill_context_parallel_size > 1:
|
||||
vllm_config = copy.copy(vllm_config)
|
||||
vllm_config.parallel_config = replace(
|
||||
parallel_config,
|
||||
prefill_context_parallel_size=1,
|
||||
)
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
vllm_config = copy.copy(vllm_config)
|
||||
vllm_config.parallel_config = replace(
|
||||
parallel_config,
|
||||
prefill_context_parallel_size=1,
|
||||
decode_context_parallel_size=(
|
||||
parallel_config.decode_context_parallel_size
|
||||
if speculative_config.draft_model_config.use_mla
|
||||
else 1
|
||||
),
|
||||
)
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
self.hidden_states = torch.zeros(
|
||||
@@ -427,7 +433,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.block_tables.input_block_tables[gid],
|
||||
self.block_tables.kernel_block_sizes[gid],
|
||||
self.block_tables.cp_rank,
|
||||
self.block_tables.cp_size,
|
||||
self.dcp_size,
|
||||
self.block_tables.cp_interleave,
|
||||
self.parallel_drafting_token_id,
|
||||
self.num_query_per_req,
|
||||
|
||||
@@ -138,6 +138,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
# DP configuration
|
||||
self.dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
|
||||
self.eplb_state: EplbState | None = None
|
||||
|
||||
@@ -332,14 +333,14 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
out=draft_seq_lens_cpu_upper_bound[:num_reqs],
|
||||
)
|
||||
draft_seq_lens_cpu_upper_bound[:num_reqs].clamp_(max=self.max_model_len)
|
||||
if dcp_local_seq_lens is None and self.block_tables.cp_size > 1:
|
||||
if dcp_local_seq_lens is None and self.dcp_size > 1:
|
||||
# Draft steps advance and rewind their own global sequence lengths,
|
||||
# so the target model's DCP-local lengths may already be stale.
|
||||
dcp_local_seq_lens = prepare_dcp_local_seq_lens(
|
||||
self.input_buffers.dcp_local_seq_lens,
|
||||
self.input_buffers.seq_lens,
|
||||
num_reqs,
|
||||
self.block_tables.cp_size,
|
||||
self.dcp_size,
|
||||
self.block_tables.cp_rank,
|
||||
self.block_tables.cp_interleave,
|
||||
)
|
||||
|
||||
@@ -817,6 +817,7 @@ class InputBatch:
|
||||
self.allowed_token_ids_mask_cpu_tensor[empty_index] = (
|
||||
self.allowed_token_ids_mask_cpu_tensor[last_req_index]
|
||||
)
|
||||
self.allowed_token_ids_mask_cpu_tensor[last_req_index].fill_(False)
|
||||
|
||||
bad_words_token_ids = self.bad_words_token_ids.pop(last_req_index, None)
|
||||
if bad_words_token_ids is not None:
|
||||
|
||||
@@ -801,11 +801,20 @@ class Worker(WorkerBase):
|
||||
# related to kv cache connector (e.g. kv cache sharing layers).
|
||||
ensure_kv_transfer_initialized(self.vllm_config, kv_cache_config)
|
||||
|
||||
# If the connector provides a custom memory pool (e.g. Mooncake
|
||||
# NVLink/BAREX), use it for KV cache allocation; otherwise fall
|
||||
# back to the standard CuMem pool.
|
||||
mem_pool_context = (
|
||||
get_kv_transfer_group().get_mem_pool_context()
|
||||
if has_kv_transfer_group()
|
||||
else None
|
||||
)
|
||||
if mem_pool_context is None:
|
||||
mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache")
|
||||
|
||||
self.model_runner.initialize_kv_cache(
|
||||
kv_cache_config,
|
||||
kv_cache_allocation_context=self._maybe_get_memory_pool_context(
|
||||
tag="kv_cache"
|
||||
),
|
||||
kv_cache_allocation_context=mem_pool_context,
|
||||
)
|
||||
|
||||
# Build KV-zero metadata outside the CuMem pool so the bookkeeping
|
||||
|
||||
Reference in New Issue
Block a user