[Bugfix][HiSparse] Preserve per-layer offsets in KV cache bindings (#57027)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-09-15 20:27:11 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 18f8aa0465
commit 38ca7a899c
4 changed files with 48 additions and 6 deletions
@@ -0,0 +1,20 @@
model_name: "nvidia/GLM-5.2-NVFP4"
accuracy_threshold: 0.90
tolerance: 0.0
num_questions: 1319
num_fewshot: 5
max_tokens: 256
max_concurrency: 32
request_timeout_seconds: 1800
startup_max_wait_seconds: 1200
server_args: >-
--tensor-parallel-size 4
--max-model-len 8192
--max-num-seqs 32
--max-num-batched-tokens 4096
--enforce-eager
--kv-cache-dtype bfloat16
--attention-config '{"hisparse_config":{}}'
--kv-transfer-config '{"kv_connector":"HiSparseConnector","kv_role":"kv_both","kv_connector_extra_config":{"host_pool_gib":32}}'
env:
VLLM_USE_V2_MODEL_RUNNER: "1"
@@ -0,0 +1 @@
GLM-5.2-NVFP4-HiSparse.yaml