[Feature][Frontend] Expose effective attention block size for DCP (#56538)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Signed-off-by: Julien Darve <jdarve@NVIDIA.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
OpenAI Codex
parent
0d8173d153
commit
f37c550bf6
@@ -215,6 +215,9 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
|
||||
base_hash = CacheConfig().compute_hash()
|
||||
assert CacheConfig(kv_cache_memory_bytes=1 << 30).compute_hash() == base_hash
|
||||
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
|
||||
config = CacheConfig()
|
||||
config.effective_attention_block_size = 64
|
||||
assert config.compute_hash() == base_hash
|
||||
|
||||
|
||||
def test_scheduler_config_hash_includes_max_num_seqs():
|
||||
|
||||
@@ -279,6 +279,44 @@ def make_request(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp", [1, 4])
|
||||
def test_effective_attention_block_size_matches_events(dcp):
|
||||
from vllm.distributed.kv_events import BlockStored
|
||||
from vllm.v1.engine.core import EngineCore
|
||||
|
||||
config = KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["attention"], new_kv_cache_spec()),
|
||||
],
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
generate_scheduler_kv_cache_config([config]),
|
||||
max_model_len=256,
|
||||
scheduler_block_size=16 * dcp,
|
||||
hash_block_size=16 * dcp,
|
||||
dcp_world_size=dcp,
|
||||
enable_kv_cache_events=True,
|
||||
)
|
||||
core = EngineCore.__new__(EngineCore)
|
||||
core.vllm_config = SimpleNamespace(cache_config=CacheConfig(block_size=16))
|
||||
core.scheduler = SimpleNamespace(kv_cache_manager=manager)
|
||||
core._initialize_effective_attention_block_size()
|
||||
block_size = core.vllm_config.cache_config.effective_attention_block_size
|
||||
assert block_size == 16 * dcp
|
||||
|
||||
request = make_request(
|
||||
"block-size", list(range(64)), block_size=16 * dcp, hash_fn=sha256
|
||||
)
|
||||
assert manager.allocate_slots(request, 64) is not None
|
||||
assert [
|
||||
event.block_size
|
||||
for event in manager.take_events()
|
||||
if isinstance(event, BlockStored)
|
||||
] == [block_size]
|
||||
|
||||
|
||||
def new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
|
||||
@@ -158,7 +158,10 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
|
||||
local_engines_only=False,
|
||||
enable_elastic_ep=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(parallel_config=parallel_config)
|
||||
vllm_config = SimpleNamespace(
|
||||
parallel_config=parallel_config,
|
||||
cache_config=SimpleNamespace(effective_attention_block_size=None),
|
||||
)
|
||||
|
||||
client = core_client_mod.MPClient(
|
||||
asyncio_mode=False,
|
||||
@@ -368,10 +371,15 @@ def test_dplb_finished_requests_release_inflight():
|
||||
assert req.request_id not in client.reqs_in_flight
|
||||
|
||||
|
||||
def test_apply_ready_response_syncs_block_size():
|
||||
@pytest.mark.parametrize(
|
||||
("effective_size", "other_size"),
|
||||
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
|
||||
)
|
||||
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
import msgspec
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
client._effective_attention_block_sizes = set()
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
@@ -399,14 +407,31 @@ def test_apply_ready_response_syncs_block_size():
|
||||
max_loras=0,
|
||||
)
|
||||
)
|
||||
client._apply_ready_response(payload)
|
||||
fields = msgspec.msgpack.decode(payload)
|
||||
if effective_size is None:
|
||||
del fields["effective_attention_block_size"]
|
||||
else:
|
||||
fields["effective_attention_block_size"] = effective_size
|
||||
client._apply_ready_response(msgspec.msgpack.encode(fields))
|
||||
assert client.vllm_config.cache_config.block_size == 1056
|
||||
cache_config = client.vllm_config.cache_config
|
||||
assert cache_config.effective_attention_block_size == effective_size
|
||||
|
||||
fields["effective_attention_block_size"] = other_size
|
||||
client._apply_ready_response(msgspec.msgpack.encode(fields))
|
||||
assert cache_config.effective_attention_block_size == (
|
||||
effective_size if effective_size == other_size else None
|
||||
)
|
||||
|
||||
client._apply_ready_response(b"")
|
||||
assert cache_config.effective_attention_block_size is None
|
||||
|
||||
|
||||
def test_apply_ready_response_syncs_mamba_block_size():
|
||||
import msgspec
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
client._effective_attention_block_sizes = set()
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
|
||||
Reference in New Issue
Block a user