[Feature][Frontend] Expose effective attention block size for DCP (#56538)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Signed-off-by: Julien Darve <jdarve@NVIDIA.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
OpenAI Codex
parent
0d8173d153
commit
f37c550bf6
@@ -20,6 +20,15 @@ Both approaches are under active development.
|
||||
|
||||
Due to the auto-regressive nature of decoding, every decoding step needs to compute a small amount of query tokens w.r.t. a large number of key/value tokens stored in the paged KV cache. The core of decode context parallel is how to shard the KV cache across GPUs.
|
||||
|
||||
`engine_client.vllm_config.cache_config.effective_attention_block_size` reports the
|
||||
initialized full-attention block size in tokens, including DCP: a physical block of
|
||||
16 tokens with DCP=4 represents 64 tokens. The same value is available through
|
||||
gRPC `Control.GetServerInfo.effective_attention_block_size`. Physical block-size
|
||||
fields retain their existing meaning. The new value is unavailable (`None` in
|
||||
Python, absent in gRPC) if there is no common full-attention size, the scheduler
|
||||
does not expose it, or an older engine omits it. gRPC also omits the value when
|
||||
engine replicas disagree. Partial cache events carry their own actual size.
|
||||
|
||||
For a model with `H` kv-heads, a request with `T` tokens in the context needs to store `H * T` key/value tensors in the KV cache.
|
||||
|
||||
1. If one GPU can hold them all, and the performance is good enough, then no parallelization is needed.
|
||||
|
||||
+1
-1
@@ -147,7 +147,7 @@ vllm-llm = { path = "src/llm" }
|
||||
vllm-managed-engine = { path = "src/managed-engine" }
|
||||
vllm-metrics = { path = "src/metrics" }
|
||||
vllm-parser = { path = "src/parser" }
|
||||
vllm-proto = { version = "0.2.0", path = "proto" }
|
||||
vllm-proto = { path = "proto" }
|
||||
vllm-server = { path = "src/server" }
|
||||
vllm-text = { path = "src/text" }
|
||||
vllm-tokenizer = { path = "src/tokenizer" }
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
This directory is the canonical source for vLLM's gRPC schema.
|
||||
|
||||
See [context parallel deployment](../../docs/serving/context_parallel_deployment.md)
|
||||
for effective attention block-size metadata in Python and `Control.GetServerInfo`.
|
||||
|
||||
Schema updates are no longer published to the Buf Schema Registry. Rust consumers
|
||||
should use `vllm-proto` from crates.io; consumers in other languages can generate
|
||||
bindings from the `.proto` files in this directory. Buf still builds and lints
|
||||
@@ -34,8 +37,7 @@ On pull requests and releases, `cargo-semver-checks` compares the crate with its
|
||||
latest published version. Include any required version bump in the protocol
|
||||
change PR. This check becomes available after the first manual publication.
|
||||
|
||||
1. Update the crate version and the `vllm-proto` workspace dependency together,
|
||||
and update `rust/Cargo.lock`.
|
||||
1. Update the crate version in `rust/proto/Cargo.toml` and update `rust/Cargo.lock`.
|
||||
2. Run `cargo publish --manifest-path rust/proto/Cargo.toml --locked --dry-run`
|
||||
and the frontend gRPC tests. Record the tested vLLM releases or revisions in
|
||||
the release notes; matching crate versions alone do not establish runtime compatibility.
|
||||
|
||||
@@ -50,6 +50,8 @@ message ServerInfo {
|
||||
uint64 max_batched_tokens = 9;
|
||||
uint32 max_loras = 10;
|
||||
RlCapabilities rl_capabilities = 11;
|
||||
// Full-attention block size in tokens after initialization; absent if unavailable.
|
||||
optional uint64 effective_attention_block_size = 14;
|
||||
}
|
||||
|
||||
message RlCapabilities {
|
||||
|
||||
@@ -510,6 +510,15 @@ impl EngineCoreClient {
|
||||
self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum()
|
||||
}
|
||||
|
||||
/// Return the effective attention block size if all engines report the same value.
|
||||
pub fn effective_attention_block_size(&self) -> Option<u64> {
|
||||
let size = self.ready_response().effective_attention_block_size?;
|
||||
self.engines
|
||||
.iter()
|
||||
.all(|engine| engine.ready_response.effective_attention_block_size == Some(size))
|
||||
.then_some(size)
|
||||
}
|
||||
|
||||
/// Return the minimum engine-reported `max_model_len` across all engines.
|
||||
///
|
||||
/// This is the auto-fitted value after KV cache profiling and may differ
|
||||
|
||||
@@ -72,6 +72,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
||||
weight_transfer_backend: None,
|
||||
enable_sleep_mode: false,
|
||||
supports_draft_weight_updates: false,
|
||||
effective_attention_block_size: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -104,6 +104,9 @@ pub struct EngineCoreReadyResponse {
|
||||
/// Whether the engine has a speculative draft model that can be updated.
|
||||
#[serde(default)]
|
||||
pub supports_draft_weight_updates: bool,
|
||||
/// Full-attention block size in tokens after initialization, or unavailable.
|
||||
#[serde(default)]
|
||||
pub effective_attention_block_size: Option<u64>,
|
||||
}
|
||||
|
||||
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
|
||||
|
||||
@@ -2846,6 +2846,11 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
|
||||
let ready_response: EngineCoreReadyResponse =
|
||||
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
|
||||
let mut legacy_ready = serde_json::to_value(&ready_response).unwrap();
|
||||
legacy_ready.as_object_mut().unwrap().remove("effective_attention_block_size");
|
||||
let legacy_ready: EngineCoreReadyResponse =
|
||||
rmp_serde::from_slice(&rmp_serde::to_vec_named(&legacy_ready).unwrap()).unwrap();
|
||||
assert!(legacy_ready.effective_attention_block_size.is_none());
|
||||
assert!(ready_response.supports_lora);
|
||||
assert_eq!(ready_response.max_loras, 8);
|
||||
assert_eq!(
|
||||
@@ -2854,6 +2859,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
);
|
||||
assert!(ready_response.enable_sleep_mode);
|
||||
assert!(ready_response.supports_draft_weight_updates);
|
||||
assert_eq!(ready_response.effective_attention_block_size, Some(64));
|
||||
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
|
||||
assert!(kv_events_config.enable_kv_cache_events);
|
||||
assert_eq!(kv_events_config.publisher, "zmq");
|
||||
|
||||
@@ -423,6 +423,7 @@ class EngineCoreReadyResponse:
|
||||
weight_transfer_backend: str | None = None
|
||||
enable_sleep_mode: bool = False
|
||||
supports_draft_weight_updates: bool = False
|
||||
effective_attention_block_size: int | None = None
|
||||
|
||||
|
||||
ready_response = EngineCoreReadyResponse(
|
||||
@@ -446,6 +447,7 @@ ready_response = EngineCoreReadyResponse(
|
||||
weight_transfer_backend="nccl",
|
||||
enable_sleep_mode=True,
|
||||
supports_draft_weight_updates=True,
|
||||
effective_attention_block_size=64,
|
||||
kv_events_config=KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
|
||||
@@ -195,6 +195,7 @@ impl pb::control_server::Control for ControlServiceImpl {
|
||||
max_batched_tokens: ready.max_num_batched_tokens,
|
||||
max_loras: ready.max_loras,
|
||||
rl_capabilities: Some(self.rl_capabilities()),
|
||||
effective_attention_block_size: self.client().effective_attention_block_size(),
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
@@ -1911,8 +1911,16 @@ async fn control_abort_resolves_external_id_and_empty_is_noop() {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn control_reports_server_and_model_info() {
|
||||
let mut ready = default_ready_response();
|
||||
ready.effective_attention_block_size = Some(64);
|
||||
let (generate_service, control_service, engine_health, _engine_task) =
|
||||
setup_grpc_service(b"engine-grpc-info", default_stream_output_specs()).await;
|
||||
setup_grpc_service_with_engine_script(
|
||||
b"engine-grpc-info".to_vec(),
|
||||
ready,
|
||||
Arc::new(FakeTextBackend),
|
||||
|_, _| boxed_test_future(async {}),
|
||||
)
|
||||
.await;
|
||||
let (channel, server_task) = start_grpc_test_server(
|
||||
generate_service,
|
||||
control_service,
|
||||
@@ -1928,6 +1936,7 @@ async fn control_reports_server_and_model_info() {
|
||||
.expect("get server info")
|
||||
.into_inner();
|
||||
assert_eq!(server.engine_version, "test-vllm-version");
|
||||
assert_eq!(server.effective_attention_block_size, Some(64));
|
||||
assert_eq!(server.api_version, "vllm");
|
||||
assert_eq!(server.instance_id, "test-instance");
|
||||
assert_eq!(server.max_model_len, DEFAULT_MOCK_MAX_MODEL_LEN as u32);
|
||||
@@ -2195,6 +2204,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
ready_0.weight_transfer_backend = Some("nccl".to_string());
|
||||
ready_0.enable_sleep_mode = true;
|
||||
ready_0.supports_draft_weight_updates = true;
|
||||
ready_0.effective_attention_block_size = Some(64);
|
||||
|
||||
let mut ready_1 = default_ready_response();
|
||||
ready_1.max_model_len = 4_096;
|
||||
@@ -2248,6 +2258,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
.into_inner();
|
||||
assert_eq!(server.max_model_len, 4_096);
|
||||
assert_eq!(server.total_kv_blocks, 30);
|
||||
assert!(server.effective_attention_block_size.is_none());
|
||||
let rl = server.rl_capabilities.expect("RL capabilities");
|
||||
assert!(!rl.weight_transfer_enabled);
|
||||
assert!(rl.weight_transfer_backend.is_empty());
|
||||
|
||||
@@ -215,6 +215,9 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
|
||||
base_hash = CacheConfig().compute_hash()
|
||||
assert CacheConfig(kv_cache_memory_bytes=1 << 30).compute_hash() == base_hash
|
||||
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
|
||||
config = CacheConfig()
|
||||
config.effective_attention_block_size = 64
|
||||
assert config.compute_hash() == base_hash
|
||||
|
||||
|
||||
def test_scheduler_config_hash_includes_max_num_seqs():
|
||||
|
||||
@@ -279,6 +279,44 @@ def make_request(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp", [1, 4])
|
||||
def test_effective_attention_block_size_matches_events(dcp):
|
||||
from vllm.distributed.kv_events import BlockStored
|
||||
from vllm.v1.engine.core import EngineCore
|
||||
|
||||
config = KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["attention"], new_kv_cache_spec()),
|
||||
],
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
generate_scheduler_kv_cache_config([config]),
|
||||
max_model_len=256,
|
||||
scheduler_block_size=16 * dcp,
|
||||
hash_block_size=16 * dcp,
|
||||
dcp_world_size=dcp,
|
||||
enable_kv_cache_events=True,
|
||||
)
|
||||
core = EngineCore.__new__(EngineCore)
|
||||
core.vllm_config = SimpleNamespace(cache_config=CacheConfig(block_size=16))
|
||||
core.scheduler = SimpleNamespace(kv_cache_manager=manager)
|
||||
core._initialize_effective_attention_block_size()
|
||||
block_size = core.vllm_config.cache_config.effective_attention_block_size
|
||||
assert block_size == 16 * dcp
|
||||
|
||||
request = make_request(
|
||||
"block-size", list(range(64)), block_size=16 * dcp, hash_fn=sha256
|
||||
)
|
||||
assert manager.allocate_slots(request, 64) is not None
|
||||
assert [
|
||||
event.block_size
|
||||
for event in manager.take_events()
|
||||
if isinstance(event, BlockStored)
|
||||
] == [block_size]
|
||||
|
||||
|
||||
def new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
|
||||
@@ -158,7 +158,10 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
|
||||
local_engines_only=False,
|
||||
enable_elastic_ep=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(parallel_config=parallel_config)
|
||||
vllm_config = SimpleNamespace(
|
||||
parallel_config=parallel_config,
|
||||
cache_config=SimpleNamespace(effective_attention_block_size=None),
|
||||
)
|
||||
|
||||
client = core_client_mod.MPClient(
|
||||
asyncio_mode=False,
|
||||
@@ -368,10 +371,15 @@ def test_dplb_finished_requests_release_inflight():
|
||||
assert req.request_id not in client.reqs_in_flight
|
||||
|
||||
|
||||
def test_apply_ready_response_syncs_block_size():
|
||||
@pytest.mark.parametrize(
|
||||
("effective_size", "other_size"),
|
||||
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
|
||||
)
|
||||
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
import msgspec
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
client._effective_attention_block_sizes = set()
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
@@ -399,14 +407,31 @@ def test_apply_ready_response_syncs_block_size():
|
||||
max_loras=0,
|
||||
)
|
||||
)
|
||||
client._apply_ready_response(payload)
|
||||
fields = msgspec.msgpack.decode(payload)
|
||||
if effective_size is None:
|
||||
del fields["effective_attention_block_size"]
|
||||
else:
|
||||
fields["effective_attention_block_size"] = effective_size
|
||||
client._apply_ready_response(msgspec.msgpack.encode(fields))
|
||||
assert client.vllm_config.cache_config.block_size == 1056
|
||||
cache_config = client.vllm_config.cache_config
|
||||
assert cache_config.effective_attention_block_size == effective_size
|
||||
|
||||
fields["effective_attention_block_size"] = other_size
|
||||
client._apply_ready_response(msgspec.msgpack.encode(fields))
|
||||
assert cache_config.effective_attention_block_size == (
|
||||
effective_size if effective_size == other_size else None
|
||||
)
|
||||
|
||||
client._apply_ready_response(b"")
|
||||
assert cache_config.effective_attention_block_size is None
|
||||
|
||||
|
||||
def test_apply_ready_response_syncs_mamba_block_size():
|
||||
import msgspec
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
client._effective_attention_block_sizes = set()
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
|
||||
@@ -211,6 +211,8 @@ class CacheConfig:
|
||||
"""The number of blocks to allocate for CPU memory."""
|
||||
|
||||
# Set after KV cache initialization.
|
||||
effective_attention_block_size: int | None = field(default=None, init=False)
|
||||
"""Full-attention block size in tokens, including DCP, or None if unavailable."""
|
||||
kv_cache_size_tokens: int | None = field(default=None, init=False)
|
||||
"""Per-DP-engine KV cache capacity in tokens (group-aware). Uses
|
||||
group-aware capacity since num_gpu_blocks * block_size can be wrong
|
||||
@@ -277,6 +279,7 @@ class CacheConfig:
|
||||
# Post-init/derived counters
|
||||
"num_gpu_blocks",
|
||||
"num_cpu_blocks",
|
||||
"effective_attention_block_size",
|
||||
"kv_cache_size_tokens",
|
||||
"kv_cache_max_concurrency",
|
||||
# WIP feature toggle not impacting compiled graph shape
|
||||
|
||||
@@ -102,6 +102,8 @@ class EngineCoreReadyResponse:
|
||||
weight_transfer_backend: str | None = None
|
||||
enable_sleep_mode: bool = False
|
||||
supports_draft_weight_updates: bool = False
|
||||
# Full-attention block size in tokens after initialization, or unavailable.
|
||||
effective_attention_block_size: int | None = None
|
||||
|
||||
|
||||
class EngineCoreRequest(
|
||||
|
||||
+20
-1
@@ -85,7 +85,7 @@ from vllm.v1.fault_tolerance.engine_core_sentinel import (
|
||||
EngineCoreSentinel,
|
||||
fault_tolerant_wrapper,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec
|
||||
from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
@@ -168,6 +168,7 @@ class EngineCore:
|
||||
block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
self._initialize_effective_attention_block_size()
|
||||
self.use_spec_decode = vllm_config.speculative_config is not None
|
||||
self.check_for_draft_tokens = (
|
||||
self.use_spec_decode or vllm_config.model_config.is_diffusion
|
||||
@@ -382,6 +383,21 @@ class EngineCore:
|
||||
)
|
||||
return scheduler_kv_cache_config
|
||||
|
||||
def _initialize_effective_attention_block_size(self) -> None:
|
||||
cache_config = self.vllm_config.cache_config
|
||||
cache_config.effective_attention_block_size = None
|
||||
cache_manager = getattr(self.scheduler, "kv_cache_manager", None)
|
||||
if cache_manager is None:
|
||||
return
|
||||
block_sizes = {
|
||||
manager.block_size
|
||||
for manager in cache_manager.coordinator.single_type_managers
|
||||
if is_full_attention_spec(manager.kv_cache_spec)
|
||||
}
|
||||
cache_config.effective_attention_block_size = (
|
||||
block_sizes.pop() if len(block_sizes) == 1 else None
|
||||
)
|
||||
|
||||
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
supported_tasks = self.model_executor.supported_tasks
|
||||
self._log_pooler_config(supported_tasks)
|
||||
@@ -1649,6 +1665,9 @@ class EngineCoreProc(EngineCore):
|
||||
num_gpu_blocks=self.vllm_config.cache_config.num_gpu_blocks or 0,
|
||||
block_size=self.vllm_config.cache_config.block_size,
|
||||
mamba_block_size=self.vllm_config.cache_config.mamba_block_size,
|
||||
effective_attention_block_size=(
|
||||
self.vllm_config.cache_config.effective_attention_block_size
|
||||
),
|
||||
dp_stats_address=self.frontend_stats_publish_address,
|
||||
dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
|
||||
vllm_version=VLLM_VERSION,
|
||||
|
||||
@@ -566,6 +566,7 @@ class MPClient(EngineCoreClient):
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self._renderer: BaseRenderer | None = renderer
|
||||
self._effective_attention_block_sizes: set[int | None] = set()
|
||||
|
||||
# ZMQ setup.
|
||||
sync_ctx = zmq.Context(io_threads=2)
|
||||
@@ -803,10 +804,21 @@ class MPClient(EngineCoreClient):
|
||||
def _apply_ready_response(self, payload: bytes) -> None:
|
||||
"""Decode an EngineCoreReadyResponse and sync any post-initialization
|
||||
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
|
||||
if not payload:
|
||||
return
|
||||
vllm_config = self.vllm_config
|
||||
response = msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
|
||||
response = (
|
||||
msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
|
||||
if payload
|
||||
else None
|
||||
)
|
||||
self._effective_attention_block_sizes.add(
|
||||
response.effective_attention_block_size if response is not None else None
|
||||
)
|
||||
sizes = self._effective_attention_block_sizes
|
||||
vllm_config.cache_config.effective_attention_block_size = (
|
||||
next(iter(sizes)) if len(sizes) == 1 else None
|
||||
)
|
||||
if response is None:
|
||||
return
|
||||
vllm_config.model_config.max_model_len = min(
|
||||
vllm_config.model_config.max_model_len, response.max_model_len
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user