[PD][Bugfix] Fix KV Cache sharing with HMA (#44629)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -35,6 +35,8 @@ sw_attn_configs=(
|
||||
"GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" # SW model
|
||||
"GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
|
||||
"GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=1 VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
|
||||
# Gemma4: SW + cross-layer KV sharing
|
||||
"GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-4-E2B-it VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
|
||||
)
|
||||
|
||||
# Select config array based on DP_EP env var
|
||||
|
||||
@@ -24,6 +24,7 @@ EXPECTED_VALUES = {
|
||||
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": 0.84,
|
||||
"ibm-granite/granite-4.0-h-tiny": 0.77,
|
||||
"Qwen/Qwen3.5-0.8B": 0.33,
|
||||
"google/gemma-4-E2B-it": 0.485,
|
||||
}
|
||||
|
||||
SIMPLE_PROMPT = (
|
||||
@@ -49,17 +50,33 @@ def test_accuracy():
|
||||
"""Run the end to end accuracy test."""
|
||||
run_simple_prompt()
|
||||
|
||||
model_args = (
|
||||
f"model={MODEL_NAME},"
|
||||
f"base_url={BASE_URL}/completions,"
|
||||
f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False"
|
||||
)
|
||||
|
||||
results = lm_eval.simple_evaluate(
|
||||
model="local-completions",
|
||||
model_args=model_args,
|
||||
tasks=TASK,
|
||||
)
|
||||
if "gemma-4" in MODEL_NAME:
|
||||
# Gemma4 is quite sensible to having a chat template applied, so we evaluate
|
||||
# on chat completions.
|
||||
model_args = (
|
||||
f"model={MODEL_NAME},"
|
||||
f"base_url={BASE_URL}/chat/completions,"
|
||||
f"num_concurrent={NUM_CONCURRENT},"
|
||||
"tokenizer_backend=huggingface"
|
||||
)
|
||||
results = lm_eval.simple_evaluate(
|
||||
model="local-chat-completions",
|
||||
model_args=model_args,
|
||||
tasks=TASK,
|
||||
num_fewshot=5,
|
||||
apply_chat_template=True,
|
||||
)
|
||||
else:
|
||||
model_args = (
|
||||
f"model={MODEL_NAME},"
|
||||
f"base_url={BASE_URL}/completions,"
|
||||
f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False"
|
||||
)
|
||||
results = lm_eval.simple_evaluate(
|
||||
model="local-completions",
|
||||
model_args=model_args,
|
||||
tasks=TASK,
|
||||
)
|
||||
|
||||
measured_value = results["results"][TASK][FILTER]
|
||||
expected_value = EXPECTED_VALUES.get(MODEL_NAME)
|
||||
|
||||
@@ -850,7 +850,14 @@ class NixlConnectorWorker:
|
||||
# However, physical page_size may differ when kernel requires a specific
|
||||
# block size. This leads to SSM and FA layers having different num_blocks.
|
||||
# `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this.
|
||||
layer_spec = self._layer_specs[layer_name]
|
||||
layer_spec = self._layer_specs.get(layer_name)
|
||||
if layer_spec is None:
|
||||
logger.debug(
|
||||
"Skipping layer %s as no KVCache spec is present. "
|
||||
"This is likely because the layer is sharing its KV cache",
|
||||
layer_name,
|
||||
)
|
||||
continue
|
||||
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
|
||||
# MLA DSv32 Indexer case: UniformTypeKVCacheSpecs merges kv_cache_specs
|
||||
layer_spec = layer_spec.kv_cache_specs[layer_name]
|
||||
|
||||
Reference in New Issue
Block a user