From 6690a0c4de37ac965475a6ea90ad4271de77a767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Tue, 9 Jun 2026 15:10:06 +0200 Subject: [PATCH] [PD][Bugfix] Fix KV Cache sharing with HMA (#44629) Signed-off-by: NickLucche --- .../config_sweep_accuracy_test.sh | 2 + .../nixl_integration/test_accuracy.py | 39 +++++++++++++------ .../kv_connector/v1/nixl/worker.py | 9 ++++- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh index 156c03d06c..432e7de3e9 100755 --- a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +++ b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh @@ -35,6 +35,8 @@ sw_attn_configs=( "GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" # SW model "GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" "GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=1 VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" + # Gemma4: SW + cross-layer KV sharing + "GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-4-E2B-it VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" ) # Select config array based on DP_EP env var diff --git a/tests/v1/kv_connector/nixl_integration/test_accuracy.py b/tests/v1/kv_connector/nixl_integration/test_accuracy.py index 036bce88e6..eead3de153 100644 --- a/tests/v1/kv_connector/nixl_integration/test_accuracy.py +++ b/tests/v1/kv_connector/nixl_integration/test_accuracy.py @@ -24,6 +24,7 @@ EXPECTED_VALUES = { "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": 0.84, "ibm-granite/granite-4.0-h-tiny": 0.77, "Qwen/Qwen3.5-0.8B": 0.33, + "google/gemma-4-E2B-it": 0.485, } SIMPLE_PROMPT = ( @@ -49,17 +50,33 @@ def test_accuracy(): """Run the end to end accuracy test.""" run_simple_prompt() - model_args = ( - f"model={MODEL_NAME}," - f"base_url={BASE_URL}/completions," - f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False" - ) - - results = lm_eval.simple_evaluate( - model="local-completions", - model_args=model_args, - tasks=TASK, - ) + if "gemma-4" in MODEL_NAME: + # Gemma4 is quite sensible to having a chat template applied, so we evaluate + # on chat completions. + model_args = ( + f"model={MODEL_NAME}," + f"base_url={BASE_URL}/chat/completions," + f"num_concurrent={NUM_CONCURRENT}," + "tokenizer_backend=huggingface" + ) + results = lm_eval.simple_evaluate( + model="local-chat-completions", + model_args=model_args, + tasks=TASK, + num_fewshot=5, + apply_chat_template=True, + ) + else: + model_args = ( + f"model={MODEL_NAME}," + f"base_url={BASE_URL}/completions," + f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False" + ) + results = lm_eval.simple_evaluate( + model="local-completions", + model_args=model_args, + tasks=TASK, + ) measured_value = results["results"][TASK][FILTER] expected_value = EXPECTED_VALUES.get(MODEL_NAME) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py index a297058c84..806a87c582 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py @@ -850,7 +850,14 @@ class NixlConnectorWorker: # However, physical page_size may differ when kernel requires a specific # block size. This leads to SSM and FA layers having different num_blocks. # `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this. - layer_spec = self._layer_specs[layer_name] + layer_spec = self._layer_specs.get(layer_name) + if layer_spec is None: + logger.debug( + "Skipping layer %s as no KVCache spec is present. " + "This is likely because the layer is sharing its KV cache", + layer_name, + ) + continue if isinstance(layer_spec, UniformTypeKVCacheSpecs): # MLA DSv32 Indexer case: UniformTypeKVCacheSpecs merges kv_cache_specs layer_spec = layer_spec.kv_cache_specs[layer_name]