[Core] Add MRV2 virtual-batch PCP for MLA (#46570)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-07-19 02:53:15 +00:00
committed by GitHub
co-authored by Codex OpenAI Codex Matthew Bonanni
parent 9243e0124e
commit b6ff8a2f50
39 changed files with 1204 additions and 196 deletions
@@ -0,0 +1,16 @@
model_name: "nvidia/GLM-5.2-NVFP4"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
max_concurrency: 100
server_args: >-
--enforce-eager
--max-model-len 4096
--safetensors-load-strategy prefetch
--moe-backend flashinfer_cutlass
--prefill-context-parallel-size 4
--enable-expert-parallel
--kv-cache-dtype fp8
env:
VLLM_LOGGING_LEVEL: "DEBUG"
VLLM_USE_V2_MODEL_RUNNER: "1"
@@ -0,0 +1,17 @@
model_name: "nvidia/GLM-5.2-NVFP4"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
max_concurrency: 100
server_args: >-
--enforce-eager
--max-model-len 4096
--safetensors-load-strategy prefetch
--moe-backend flashinfer_cutlass
--tensor-parallel-size 2
--prefill-context-parallel-size 2
--enable-expert-parallel
--kv-cache-dtype fp8
env:
VLLM_LOGGING_LEVEL: "DEBUG"
VLLM_USE_V2_MODEL_RUNNER: "1"
+2
View File
@@ -0,0 +1,2 @@
GLM-5.2-NVFP4-TP2-PCP2-EP.yaml
GLM-5.2-NVFP4-TP1-PCP4-EP.yaml
+15 -1
View File
@@ -217,6 +217,7 @@ def evaluate_gsm8k(
seed: int | None = 42,
request_timeout_seconds: float = 600,
gen_prefix: str = "",
max_concurrency: int | None = None,
) -> dict[str, float | int]:
"""
Evaluate GSM8K accuracy using vLLM serve endpoint.
@@ -261,7 +262,14 @@ def evaluate_gsm8k(
return answer, tokens
timeout = aiohttp.ClientTimeout(total=request_timeout_seconds)
async with aiohttp.ClientSession(timeout=timeout) as session:
connector = (
aiohttp.TCPConnector(limit=max_concurrency)
if max_concurrency is not None
else None
)
async with aiohttp.ClientSession(
timeout=timeout, connector=connector
) as session:
tasks = [get_answer(session, i) for i in range(num_questions)]
await tqdm.gather(*tasks, desc="Evaluating")
@@ -343,6 +351,11 @@ def main() -> None:
parser.add_argument(
"--seed", type=int, default=42, help="Random seed for reproducibility"
)
parser.add_argument(
"--max-concurrency",
type=int,
help="Maximum number of concurrent requests",
)
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
args = parser.parse_args()
@@ -355,6 +368,7 @@ def main() -> None:
port=args.port,
temperature=args.temperature,
seed=args.seed,
max_concurrency=args.max_concurrency,
)
# Print results to terminal
@@ -59,6 +59,7 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
seed=eval_config.get("seed", 42),
request_timeout_seconds=request_timeout_seconds,
gen_prefix=eval_config.get("gen_prefix", ""),
max_concurrency=eval_config.get("max_concurrency"),
)
return results