diff --git a/docs/serving/context_parallel_deployment.md b/docs/serving/context_parallel_deployment.md index 3df2b06635..cb9a1ece71 100644 --- a/docs/serving/context_parallel_deployment.md +++ b/docs/serving/context_parallel_deployment.md @@ -20,6 +20,15 @@ Both approaches are under active development. Due to the auto-regressive nature of decoding, every decoding step needs to compute a small amount of query tokens w.r.t. a large number of key/value tokens stored in the paged KV cache. The core of decode context parallel is how to shard the KV cache across GPUs. +`engine_client.vllm_config.cache_config.effective_attention_block_size` reports the +initialized full-attention block size in tokens, including DCP: a physical block of +16 tokens with DCP=4 represents 64 tokens. The same value is available through +gRPC `Control.GetServerInfo.effective_attention_block_size`. Physical block-size +fields retain their existing meaning. The new value is unavailable (`None` in +Python, absent in gRPC) if there is no common full-attention size, the scheduler +does not expose it, or an older engine omits it. gRPC also omits the value when +engine replicas disagree. Partial cache events carry their own actual size. + For a model with `H` kv-heads, a request with `T` tokens in the context needs to store `H * T` key/value tensors in the KV cache. 1. If one GPU can hold them all, and the performance is good enough, then no parallelization is needed. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index bd9d99de62..dde065e790 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -147,7 +147,7 @@ vllm-llm = { path = "src/llm" } vllm-managed-engine = { path = "src/managed-engine" } vllm-metrics = { path = "src/metrics" } vllm-parser = { path = "src/parser" } -vllm-proto = { version = "0.2.0", path = "proto" } +vllm-proto = { path = "proto" } vllm-server = { path = "src/server" } vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } diff --git a/rust/proto/README.md b/rust/proto/README.md index 3181cdc6a0..b8a20b0c34 100644 --- a/rust/proto/README.md +++ b/rust/proto/README.md @@ -2,6 +2,9 @@ This directory is the canonical source for vLLM's gRPC schema. +See [context parallel deployment](../../docs/serving/context_parallel_deployment.md) +for effective attention block-size metadata in Python and `Control.GetServerInfo`. + Schema updates are no longer published to the Buf Schema Registry. Rust consumers should use `vllm-proto` from crates.io; consumers in other languages can generate bindings from the `.proto` files in this directory. Buf still builds and lints @@ -34,8 +37,7 @@ On pull requests and releases, `cargo-semver-checks` compares the crate with its latest published version. Include any required version bump in the protocol change PR. This check becomes available after the first manual publication. -1. Update the crate version and the `vllm-proto` workspace dependency together, - and update `rust/Cargo.lock`. +1. Update the crate version in `rust/proto/Cargo.toml` and update `rust/Cargo.lock`. 2. Run `cargo publish --manifest-path rust/proto/Cargo.toml --locked --dry-run` and the frontend gRPC tests. Record the tested vLLM releases or revisions in the release notes; matching crate versions alone do not establish runtime compatibility. diff --git a/rust/proto/control.proto b/rust/proto/control.proto index 3050035fdc..d2816f68d3 100644 --- a/rust/proto/control.proto +++ b/rust/proto/control.proto @@ -50,6 +50,8 @@ message ServerInfo { uint64 max_batched_tokens = 9; uint32 max_loras = 10; RlCapabilities rl_capabilities = 11; + // Full-attention block size in tokens after initialization; absent if unavailable. + optional uint64 effective_attention_block_size = 14; } message RlCapabilities { diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 63689ea3db..4cd7420e56 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -510,6 +510,15 @@ impl EngineCoreClient { self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum() } + /// Return the effective attention block size if all engines report the same value. + pub fn effective_attention_block_size(&self) -> Option { + let size = self.ready_response().effective_attention_block_size?; + self.engines + .iter() + .all(|engine| engine.ready_response.effective_attention_block_size == Some(size)) + .then_some(size) + } + /// Return the minimum engine-reported `max_model_len` across all engines. /// /// This is the auto-fitted value after KV cache profiling and may differ diff --git a/rust/src/engine-core-client/src/mock_engine.rs b/rust/src/engine-core-client/src/mock_engine.rs index 77d72b525f..958fcf8aa1 100644 --- a/rust/src/engine-core-client/src/mock_engine.rs +++ b/rust/src/engine-core-client/src/mock_engine.rs @@ -72,6 +72,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse { weight_transfer_backend: None, enable_sleep_mode: false, supports_draft_weight_updates: false, + effective_attention_block_size: None, } } diff --git a/rust/src/engine-core-client/src/protocol/handshake.rs b/rust/src/engine-core-client/src/protocol/handshake.rs index c14c19a503..52d897be52 100644 --- a/rust/src/engine-core-client/src/protocol/handshake.rs +++ b/rust/src/engine-core-client/src/protocol/handshake.rs @@ -104,6 +104,9 @@ pub struct EngineCoreReadyResponse { /// Whether the engine has a speculative draft model that can be updated. #[serde(default)] pub supports_draft_weight_updates: bool, + /// Full-attention block size in tokens after initialization, or unavailable. + #[serde(default)] + pub effective_attention_block_size: Option, } /// Frontend-owned ZMQ addresses that are sent to the engine during startup diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 27fbbe9abd..3db573d2aa 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2846,6 +2846,11 @@ fn python_msgpack_fixtures_match_rust_encoding() { let ready_response: EngineCoreReadyResponse = rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap(); + let mut legacy_ready = serde_json::to_value(&ready_response).unwrap(); + legacy_ready.as_object_mut().unwrap().remove("effective_attention_block_size"); + let legacy_ready: EngineCoreReadyResponse = + rmp_serde::from_slice(&rmp_serde::to_vec_named(&legacy_ready).unwrap()).unwrap(); + assert!(legacy_ready.effective_attention_block_size.is_none()); assert!(ready_response.supports_lora); assert_eq!(ready_response.max_loras, 8); assert_eq!( @@ -2854,6 +2859,7 @@ fn python_msgpack_fixtures_match_rust_encoding() { ); assert!(ready_response.enable_sleep_mode); assert!(ready_response.supports_draft_weight_updates); + assert_eq!(ready_response.effective_attention_block_size, Some(64)); let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode"); assert!(kv_events_config.enable_kv_cache_events); assert_eq!(kv_events_config.publisher, "zmq"); diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index 20cec81808..0cebca972d 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -423,6 +423,7 @@ class EngineCoreReadyResponse: weight_transfer_backend: str | None = None enable_sleep_mode: bool = False supports_draft_weight_updates: bool = False + effective_attention_block_size: int | None = None ready_response = EngineCoreReadyResponse( @@ -446,6 +447,7 @@ ready_response = EngineCoreReadyResponse( weight_transfer_backend="nccl", enable_sleep_mode=True, supports_draft_weight_updates=True, + effective_attention_block_size=64, kv_events_config=KVEventsConfig( enable_kv_cache_events=True, publisher="zmq", diff --git a/rust/src/server/src/grpc/control.rs b/rust/src/server/src/grpc/control.rs index ddb74bf40f..de9c5ae65f 100644 --- a/rust/src/server/src/grpc/control.rs +++ b/rust/src/server/src/grpc/control.rs @@ -195,6 +195,7 @@ impl pb::control_server::Control for ControlServiceImpl { max_batched_tokens: ready.max_num_batched_tokens, max_loras: ready.max_loras, rl_capabilities: Some(self.rl_capabilities()), + effective_attention_block_size: self.client().effective_attention_block_size(), })) } diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index f5f74d557e..a21e013323 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -1911,8 +1911,16 @@ async fn control_abort_resolves_external_id_and_empty_is_noop() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn control_reports_server_and_model_info() { + let mut ready = default_ready_response(); + ready.effective_attention_block_size = Some(64); let (generate_service, control_service, engine_health, _engine_task) = - setup_grpc_service(b"engine-grpc-info", default_stream_output_specs()).await; + setup_grpc_service_with_engine_script( + b"engine-grpc-info".to_vec(), + ready, + Arc::new(FakeTextBackend), + |_, _| boxed_test_future(async {}), + ) + .await; let (channel, server_task) = start_grpc_test_server( generate_service, control_service, @@ -1928,6 +1936,7 @@ async fn control_reports_server_and_model_info() { .expect("get server info") .into_inner(); assert_eq!(server.engine_version, "test-vllm-version"); + assert_eq!(server.effective_attention_block_size, Some(64)); assert_eq!(server.api_version, "vllm"); assert_eq!(server.instance_id, "test-instance"); assert_eq!(server.max_model_len, DEFAULT_MOCK_MAX_MODEL_LEN as u32); @@ -2195,6 +2204,7 @@ async fn control_aggregates_multi_engine_capacity() { ready_0.weight_transfer_backend = Some("nccl".to_string()); ready_0.enable_sleep_mode = true; ready_0.supports_draft_weight_updates = true; + ready_0.effective_attention_block_size = Some(64); let mut ready_1 = default_ready_response(); ready_1.max_model_len = 4_096; @@ -2248,6 +2258,7 @@ async fn control_aggregates_multi_engine_capacity() { .into_inner(); assert_eq!(server.max_model_len, 4_096); assert_eq!(server.total_kv_blocks, 30); + assert!(server.effective_attention_block_size.is_none()); let rl = server.rl_capabilities.expect("RL capabilities"); assert!(!rl.weight_transfer_enabled); assert!(rl.weight_transfer_backend.is_empty()); diff --git a/tests/config/test_config_utils.py b/tests/config/test_config_utils.py index 94060d1724..590e97c1eb 100644 --- a/tests/config/test_config_utils.py +++ b/tests/config/test_config_utils.py @@ -215,6 +215,9 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs(): base_hash = CacheConfig().compute_hash() assert CacheConfig(kv_cache_memory_bytes=1 << 30).compute_hash() == base_hash assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash + config = CacheConfig() + config.effective_attention_block_size = 64 + assert config.compute_hash() == base_hash def test_scheduler_config_hash_includes_max_num_seqs(): diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index c6354cdc0f..86e66e1941 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -279,6 +279,44 @@ def make_request( ) +@pytest.mark.parametrize("dcp", [1, 4]) +def test_effective_attention_block_size_matches_events(dcp): + from vllm.distributed.kv_events import BlockStored + from vllm.v1.engine.core import EngineCore + + config = KVCacheConfig( + num_blocks=32, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec(["attention"], new_kv_cache_spec()), + ], + ) + manager = KVCacheManager( + generate_scheduler_kv_cache_config([config]), + max_model_len=256, + scheduler_block_size=16 * dcp, + hash_block_size=16 * dcp, + dcp_world_size=dcp, + enable_kv_cache_events=True, + ) + core = EngineCore.__new__(EngineCore) + core.vllm_config = SimpleNamespace(cache_config=CacheConfig(block_size=16)) + core.scheduler = SimpleNamespace(kv_cache_manager=manager) + core._initialize_effective_attention_block_size() + block_size = core.vllm_config.cache_config.effective_attention_block_size + assert block_size == 16 * dcp + + request = make_request( + "block-size", list(range(64)), block_size=16 * dcp, hash_fn=sha256 + ) + assert manager.allocate_slots(request, 64) is not None + assert [ + event.block_size + for event in manager.take_events() + if isinstance(event, BlockStored) + ] == [block_size] + + def new_kv_cache_spec( block_size=16, num_kv_heads=2, diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index f97ed76c5c..3f407457b2 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -158,7 +158,10 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch): local_engines_only=False, enable_elastic_ep=False, ) - vllm_config = SimpleNamespace(parallel_config=parallel_config) + vllm_config = SimpleNamespace( + parallel_config=parallel_config, + cache_config=SimpleNamespace(effective_attention_block_size=None), + ) client = core_client_mod.MPClient( asyncio_mode=False, @@ -368,10 +371,15 @@ def test_dplb_finished_requests_release_inflight(): assert req.request_id not in client.reqs_in_flight -def test_apply_ready_response_syncs_block_size(): +@pytest.mark.parametrize( + ("effective_size", "other_size"), + [(None, None), (4224, 4224), (4224, 1056), (4224, None)], +) +def test_apply_ready_response_syncs_block_size(effective_size, other_size): import msgspec client = object.__new__(MPClient) + client._effective_attention_block_sizes = set() client.vllm_config = SimpleNamespace( cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0), model_config=SimpleNamespace(max_model_len=8192), @@ -399,14 +407,31 @@ def test_apply_ready_response_syncs_block_size(): max_loras=0, ) ) - client._apply_ready_response(payload) + fields = msgspec.msgpack.decode(payload) + if effective_size is None: + del fields["effective_attention_block_size"] + else: + fields["effective_attention_block_size"] = effective_size + client._apply_ready_response(msgspec.msgpack.encode(fields)) assert client.vllm_config.cache_config.block_size == 1056 + cache_config = client.vllm_config.cache_config + assert cache_config.effective_attention_block_size == effective_size + + fields["effective_attention_block_size"] = other_size + client._apply_ready_response(msgspec.msgpack.encode(fields)) + assert cache_config.effective_attention_block_size == ( + effective_size if effective_size == other_size else None + ) + + client._apply_ready_response(b"") + assert cache_config.effective_attention_block_size is None def test_apply_ready_response_syncs_mamba_block_size(): import msgspec client = object.__new__(MPClient) + client._effective_attention_block_sizes = set() client.vllm_config = SimpleNamespace( cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0), model_config=SimpleNamespace(max_model_len=8192), diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 9bd898731e..ce3199c52a 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -211,6 +211,8 @@ class CacheConfig: """The number of blocks to allocate for CPU memory.""" # Set after KV cache initialization. + effective_attention_block_size: int | None = field(default=None, init=False) + """Full-attention block size in tokens, including DCP, or None if unavailable.""" kv_cache_size_tokens: int | None = field(default=None, init=False) """Per-DP-engine KV cache capacity in tokens (group-aware). Uses group-aware capacity since num_gpu_blocks * block_size can be wrong @@ -277,6 +279,7 @@ class CacheConfig: # Post-init/derived counters "num_gpu_blocks", "num_cpu_blocks", + "effective_attention_block_size", "kv_cache_size_tokens", "kv_cache_max_concurrency", # WIP feature toggle not impacting compiled graph shape diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index 5ae9ee0cac..7d68b6e797 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -102,6 +102,8 @@ class EngineCoreReadyResponse: weight_transfer_backend: str | None = None enable_sleep_mode: bool = False supports_draft_weight_updates: bool = False + # Full-attention block size in tokens after initialization, or unavailable. + effective_attention_block_size: int | None = None class EngineCoreRequest( diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 62b6ef880e..b3d94de3e7 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -85,7 +85,7 @@ from vllm.v1.fault_tolerance.engine_core_sentinel import ( EngineCoreSentinel, fault_tolerant_wrapper, ) -from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats from vllm.v1.outputs import ModelRunnerOutput from vllm.v1.request import Request, RequestStatus @@ -168,6 +168,7 @@ class EngineCore: block_size=scheduler_block_size, hash_block_size=hash_block_size, ) + self._initialize_effective_attention_block_size() self.use_spec_decode = vllm_config.speculative_config is not None self.check_for_draft_tokens = ( self.use_spec_decode or vllm_config.model_config.is_diffusion @@ -382,6 +383,21 @@ class EngineCore: ) return scheduler_kv_cache_config + def _initialize_effective_attention_block_size(self) -> None: + cache_config = self.vllm_config.cache_config + cache_config.effective_attention_block_size = None + cache_manager = getattr(self.scheduler, "kv_cache_manager", None) + if cache_manager is None: + return + block_sizes = { + manager.block_size + for manager in cache_manager.coordinator.single_type_managers + if is_full_attention_spec(manager.kv_cache_spec) + } + cache_config.effective_attention_block_size = ( + block_sizes.pop() if len(block_sizes) == 1 else None + ) + def get_supported_tasks(self) -> tuple[SupportedTask, ...]: supported_tasks = self.model_executor.supported_tasks self._log_pooler_config(supported_tasks) @@ -1649,6 +1665,9 @@ class EngineCoreProc(EngineCore): num_gpu_blocks=self.vllm_config.cache_config.num_gpu_blocks or 0, block_size=self.vllm_config.cache_config.block_size, mamba_block_size=self.vllm_config.cache_config.mamba_block_size, + effective_attention_block_size=( + self.vllm_config.cache_config.effective_attention_block_size + ), dp_stats_address=self.frontend_stats_publish_address, dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."), vllm_version=VLLM_VERSION, diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index c354ba699c..b63e8016c4 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -566,6 +566,7 @@ class MPClient(EngineCoreClient): ): self.vllm_config = vllm_config self._renderer: BaseRenderer | None = renderer + self._effective_attention_block_sizes: set[int | None] = set() # ZMQ setup. sync_ctx = zmq.Context(io_threads=2) @@ -803,10 +804,21 @@ class MPClient(EngineCoreClient): def _apply_ready_response(self, payload: bytes) -> None: """Decode an EngineCoreReadyResponse and sync any post-initialization config changes (e.g. auto-fitted max_model_len) back to the frontend.""" - if not payload: - return vllm_config = self.vllm_config - response = msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse) + response = ( + msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse) + if payload + else None + ) + self._effective_attention_block_sizes.add( + response.effective_attention_block_size if response is not None else None + ) + sizes = self._effective_attention_block_sizes + vllm_config.cache_config.effective_attention_block_size = ( + next(iter(sizes)) if len(sizes) == 1 else None + ) + if response is None: + return vllm_config.model_config.max_model_len = min( vllm_config.model_config.max_model_len, response.max_model_len )