[Core] Add MRV2 virtual-batch PCP for MLA (#46570)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Matthew Bonanni
parent
9243e0124e
commit
b6ff8a2f50
@@ -0,0 +1,16 @@
|
||||
model_name: "nvidia/GLM-5.2-NVFP4"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 100
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--safetensors-load-strategy prefetch
|
||||
--moe-backend flashinfer_cutlass
|
||||
--prefill-context-parallel-size 4
|
||||
--enable-expert-parallel
|
||||
--kv-cache-dtype fp8
|
||||
env:
|
||||
VLLM_LOGGING_LEVEL: "DEBUG"
|
||||
VLLM_USE_V2_MODEL_RUNNER: "1"
|
||||
@@ -0,0 +1,17 @@
|
||||
model_name: "nvidia/GLM-5.2-NVFP4"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 100
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--safetensors-load-strategy prefetch
|
||||
--moe-backend flashinfer_cutlass
|
||||
--tensor-parallel-size 2
|
||||
--prefill-context-parallel-size 2
|
||||
--enable-expert-parallel
|
||||
--kv-cache-dtype fp8
|
||||
env:
|
||||
VLLM_LOGGING_LEVEL: "DEBUG"
|
||||
VLLM_USE_V2_MODEL_RUNNER: "1"
|
||||
@@ -0,0 +1,2 @@
|
||||
GLM-5.2-NVFP4-TP2-PCP2-EP.yaml
|
||||
GLM-5.2-NVFP4-TP1-PCP4-EP.yaml
|
||||
@@ -217,6 +217,7 @@ def evaluate_gsm8k(
|
||||
seed: int | None = 42,
|
||||
request_timeout_seconds: float = 600,
|
||||
gen_prefix: str = "",
|
||||
max_concurrency: int | None = None,
|
||||
) -> dict[str, float | int]:
|
||||
"""
|
||||
Evaluate GSM8K accuracy using vLLM serve endpoint.
|
||||
@@ -261,7 +262,14 @@ def evaluate_gsm8k(
|
||||
return answer, tokens
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=request_timeout_seconds)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
connector = (
|
||||
aiohttp.TCPConnector(limit=max_concurrency)
|
||||
if max_concurrency is not None
|
||||
else None
|
||||
)
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=timeout, connector=connector
|
||||
) as session:
|
||||
tasks = [get_answer(session, i) for i in range(num_questions)]
|
||||
await tqdm.gather(*tasks, desc="Evaluating")
|
||||
|
||||
@@ -343,6 +351,11 @@ def main() -> None:
|
||||
parser.add_argument(
|
||||
"--seed", type=int, default=42, help="Random seed for reproducibility"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-concurrency",
|
||||
type=int,
|
||||
help="Maximum number of concurrent requests",
|
||||
)
|
||||
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
|
||||
|
||||
args = parser.parse_args()
|
||||
@@ -355,6 +368,7 @@ def main() -> None:
|
||||
port=args.port,
|
||||
temperature=args.temperature,
|
||||
seed=args.seed,
|
||||
max_concurrency=args.max_concurrency,
|
||||
)
|
||||
|
||||
# Print results to terminal
|
||||
|
||||
@@ -59,6 +59,7 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
|
||||
seed=eval_config.get("seed", 42),
|
||||
request_timeout_seconds=request_timeout_seconds,
|
||||
gen_prefix=eval_config.get("gen_prefix", ""),
|
||||
max_concurrency=eval_config.get("max_concurrency"),
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
Reference in New Issue
Block a user