[Core] Add MRV2 virtual-batch PCP for MLA (#46570)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Matthew Bonanni
parent
9243e0124e
commit
b6ff8a2f50
@@ -0,0 +1,16 @@
|
||||
model_name: "nvidia/GLM-5.2-NVFP4"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 100
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--safetensors-load-strategy prefetch
|
||||
--moe-backend flashinfer_cutlass
|
||||
--prefill-context-parallel-size 4
|
||||
--enable-expert-parallel
|
||||
--kv-cache-dtype fp8
|
||||
env:
|
||||
VLLM_LOGGING_LEVEL: "DEBUG"
|
||||
VLLM_USE_V2_MODEL_RUNNER: "1"
|
||||
@@ -0,0 +1,17 @@
|
||||
model_name: "nvidia/GLM-5.2-NVFP4"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 100
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--safetensors-load-strategy prefetch
|
||||
--moe-backend flashinfer_cutlass
|
||||
--tensor-parallel-size 2
|
||||
--prefill-context-parallel-size 2
|
||||
--enable-expert-parallel
|
||||
--kv-cache-dtype fp8
|
||||
env:
|
||||
VLLM_LOGGING_LEVEL: "DEBUG"
|
||||
VLLM_USE_V2_MODEL_RUNNER: "1"
|
||||
@@ -0,0 +1,2 @@
|
||||
GLM-5.2-NVFP4-TP2-PCP2-EP.yaml
|
||||
GLM-5.2-NVFP4-TP1-PCP4-EP.yaml
|
||||
@@ -217,6 +217,7 @@ def evaluate_gsm8k(
|
||||
seed: int | None = 42,
|
||||
request_timeout_seconds: float = 600,
|
||||
gen_prefix: str = "",
|
||||
max_concurrency: int | None = None,
|
||||
) -> dict[str, float | int]:
|
||||
"""
|
||||
Evaluate GSM8K accuracy using vLLM serve endpoint.
|
||||
@@ -261,7 +262,14 @@ def evaluate_gsm8k(
|
||||
return answer, tokens
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=request_timeout_seconds)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
connector = (
|
||||
aiohttp.TCPConnector(limit=max_concurrency)
|
||||
if max_concurrency is not None
|
||||
else None
|
||||
)
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=timeout, connector=connector
|
||||
) as session:
|
||||
tasks = [get_answer(session, i) for i in range(num_questions)]
|
||||
await tqdm.gather(*tasks, desc="Evaluating")
|
||||
|
||||
@@ -343,6 +351,11 @@ def main() -> None:
|
||||
parser.add_argument(
|
||||
"--seed", type=int, default=42, help="Random seed for reproducibility"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-concurrency",
|
||||
type=int,
|
||||
help="Maximum number of concurrent requests",
|
||||
)
|
||||
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
|
||||
|
||||
args = parser.parse_args()
|
||||
@@ -355,6 +368,7 @@ def main() -> None:
|
||||
port=args.port,
|
||||
temperature=args.temperature,
|
||||
seed=args.seed,
|
||||
max_concurrency=args.max_concurrency,
|
||||
)
|
||||
|
||||
# Print results to terminal
|
||||
|
||||
@@ -59,6 +59,7 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
|
||||
seed=eval_config.get("seed", 42),
|
||||
request_timeout_seconds=request_timeout_seconds,
|
||||
gen_prefix=eval_config.get("gen_prefix", ""),
|
||||
max_concurrency=eval_config.get("max_concurrency"),
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
@@ -160,6 +160,8 @@ def apply_split_decodes_and_prefills(
|
||||
decode_threshold: int,
|
||||
require_uniform: bool,
|
||||
padded_num_tokens: int | None = None,
|
||||
is_prefilling: list[bool] | None = None,
|
||||
treat_short_extends_as_decodes: bool = True,
|
||||
):
|
||||
"""Helper function to apply split_decodes_and_prefills and return
|
||||
the results."""
|
||||
@@ -173,11 +175,14 @@ def apply_split_decodes_and_prefills(
|
||||
|
||||
if padded_num_tokens is not None:
|
||||
common_metadata.num_actual_tokens = padded_num_tokens
|
||||
if is_prefilling is not None:
|
||||
common_metadata.is_prefilling = torch.tensor(is_prefilling)
|
||||
|
||||
return split_decodes_and_prefills(
|
||||
common_metadata,
|
||||
decode_threshold=decode_threshold,
|
||||
require_uniform=require_uniform,
|
||||
treat_short_extends_as_decodes=treat_short_extends_as_decodes,
|
||||
)
|
||||
|
||||
|
||||
@@ -236,6 +241,17 @@ def test_split_decodes_and_prefills_uniform_all_ones():
|
||||
assert num_prefill_tokens == 0
|
||||
|
||||
|
||||
def test_split_decodes_and_prefills_uniform_short_extend():
|
||||
result = apply_split_decodes_and_prefills(
|
||||
[1, 1],
|
||||
decode_threshold=1,
|
||||
require_uniform=True,
|
||||
is_prefilling=[False, True],
|
||||
treat_short_extends_as_decodes=False,
|
||||
)
|
||||
assert result == (1, 1, 1, 1)
|
||||
|
||||
|
||||
def test_split_decodes_and_prefills_uniform_all_short_decodes():
|
||||
query_lens = [2, 2, 1, 3, 2, 1, 2]
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
|
||||
@@ -1114,6 +1114,13 @@ def test_sparse_backend_prefill_correctness(
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,query_lens,workspace_size,max_logits_bytes,expected",
|
||||
[
|
||||
(
|
||||
torch.tensor([0]),
|
||||
torch.tensor([0]),
|
||||
100,
|
||||
1000,
|
||||
[],
|
||||
),
|
||||
# Logits constraint triggers split (M*N exceeds budget)
|
||||
# req0: M=10, N=100 -> 1000 elems (4000 bytes) - fits in 5000
|
||||
# req1: adding M=10, N=100 -> new_M=20, new_N=200 -> 4000 elems > 1250
|
||||
|
||||
@@ -385,7 +385,7 @@ def test_tiering_spec_aligns_row_size():
|
||||
assert spec.num_blocks == cpu_bytes_to_use // alignment
|
||||
|
||||
|
||||
def test_offloading_spec_resolves_prefill_context_parallel_block_sizes():
|
||||
def test_offloading_spec_kv_sharding_ignores_prefill_context_parallel():
|
||||
config = _make_layout_vllm_config(
|
||||
cpu_bytes_to_use=65536,
|
||||
extra_config={"block_size": 64},
|
||||
@@ -394,9 +394,9 @@ def test_offloading_spec_resolves_prefill_context_parallel_block_sizes():
|
||||
|
||||
spec = _create_spec(config, _make_kv_cache_config())
|
||||
|
||||
assert spec.tokens_per_block == (32,)
|
||||
assert spec.tokens_per_hash == 32
|
||||
assert spec.blocks_per_chunk == 2
|
||||
assert spec.tokens_per_block == (16,)
|
||||
assert spec.tokens_per_hash == 16
|
||||
assert spec.blocks_per_chunk == 4
|
||||
|
||||
|
||||
def test_offloading_config_preserves_data_parallel_index():
|
||||
|
||||
@@ -1551,9 +1551,8 @@ def _make_cp_scheduler(
|
||||
num_gpu_blocks: int = 16,
|
||||
lazy: bool = False,
|
||||
) -> SchedulerFixture:
|
||||
"""Build a SimpleCPUOffloadScheduler with CP-scaled virtual block size."""
|
||||
cp_world_size = dcp_world_size * pcp_world_size
|
||||
virtual_block_size = BLOCK_SIZE * cp_world_size
|
||||
"""Build a SimpleCPUOffloadScheduler with DCP-scaled block size."""
|
||||
virtual_block_size = BLOCK_SIZE * dcp_world_size
|
||||
|
||||
kv_cache_config = _make_kv_cache_config(num_gpu_blocks)
|
||||
vllm_config = _make_cp_vllm_config(dcp_world_size, pcp_world_size)
|
||||
@@ -1645,17 +1644,14 @@ def _allocate_cp_gpu_blocks(
|
||||
],
|
||||
)
|
||||
def test_cp_block_size_scaling(dcp_world_size: int, pcp_world_size: int) -> None:
|
||||
"""Verify that the scheduler's block_size and cp_world_size are correctly
|
||||
scaled when context parallelism is enabled."""
|
||||
"""Verify block size scaling follows DCP ownership."""
|
||||
fix = _make_cp_scheduler(
|
||||
dcp_world_size=dcp_world_size, pcp_world_size=pcp_world_size
|
||||
)
|
||||
sched = fix.scheduler
|
||||
|
||||
expected_cp = dcp_world_size * pcp_world_size
|
||||
assert sched.cp_world_size == expected_cp
|
||||
assert sched.block_size == BLOCK_SIZE * expected_cp
|
||||
assert sched.fa_block_size == BLOCK_SIZE * expected_cp
|
||||
assert sched.cp_world_size == dcp_world_size
|
||||
assert sched.block_size == BLOCK_SIZE * dcp_world_size
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1682,8 +1678,7 @@ def test_cp_eager_store_and_load_roundtrip(
|
||||
lazy=False,
|
||||
)
|
||||
sched = fix.scheduler
|
||||
cp = dcp_world_size * pcp_world_size
|
||||
vbs = BLOCK_SIZE * cp
|
||||
vbs = BLOCK_SIZE * dcp_world_size
|
||||
|
||||
num_blocks = 2
|
||||
req = _make_cp_request(num_blocks, vbs)
|
||||
@@ -1749,18 +1744,16 @@ def test_cp_eager_store_and_load_roundtrip(
|
||||
def test_cp_effective_block_size_store_and_load(
|
||||
dcp_world_size: int, pcp_world_size: int
|
||||
) -> None:
|
||||
"""Verify ready_blocks_g (store) and n_take_g (load) use the effective
|
||||
(physical * cp) block size, not the per-rank physical size."""
|
||||
"""Verify store/load physical block counts scale with DCP, not PCP."""
|
||||
fix = _make_cp_scheduler(
|
||||
dcp_world_size=dcp_world_size, pcp_world_size=pcp_world_size
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
cp = dcp_world_size * pcp_world_size
|
||||
vbs = BLOCK_SIZE * cp
|
||||
vbs = BLOCK_SIZE * dcp_world_size
|
||||
expected_blocks = 1
|
||||
|
||||
# Store: allocate 2 blocks, confirm only 1. Without the fix,
|
||||
# ready_blocks_g = vbs / BLOCK_SIZE = 2, storing both blocks.
|
||||
# Store one DCP-scaled logical block worth of tokens.
|
||||
req = _make_cp_request(num_blocks=2, virtual_block_size=vbs)
|
||||
gpu_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs)
|
||||
kv = KVCacheBlocks(blocks=(gpu_blocks,))
|
||||
@@ -1772,12 +1765,11 @@ def test_cp_effective_block_size_store_and_load(
|
||||
new_reqs={req.request_id: kv.get_block_ids()},
|
||||
)
|
||||
)
|
||||
assert len(m1.store_gpu_blocks) == 1
|
||||
assert len(m1.store_cpu_blocks) == 1
|
||||
assert len(m1.store_gpu_blocks) == expected_blocks
|
||||
assert len(m1.store_cpu_blocks) == expected_blocks
|
||||
simulate_store_completion(sched, m1.store_event)
|
||||
|
||||
# Load: store 2 blocks from a second request, accept only 1 as external.
|
||||
# Without the fix, n_take_g = vbs / BLOCK_SIZE = 2, loading both.
|
||||
# Load one DCP-scaled logical block from a two-block CPU hit.
|
||||
req2 = _make_cp_request(num_blocks=2, virtual_block_size=vbs)
|
||||
kv2 = KVCacheBlocks(blocks=(_allocate_cp_gpu_blocks(gpu_pool, req2, 2, vbs),))
|
||||
req2.num_computed_tokens = 2 * vbs
|
||||
@@ -1801,7 +1793,7 @@ def test_cp_effective_block_size_store_and_load(
|
||||
hit, _ = sched.get_num_new_matched_tokens(req3, num_computed_tokens=0)
|
||||
assert hit == 2 * vbs
|
||||
|
||||
kv3 = KVCacheBlocks(blocks=(gpu_pool.get_new_blocks(2),))
|
||||
kv3 = KVCacheBlocks(blocks=(gpu_pool.get_new_blocks(expected_blocks),))
|
||||
sched.update_state_after_alloc(req3, kv3, num_external_tokens=vbs)
|
||||
m3 = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
@@ -1810,8 +1802,8 @@ def test_cp_effective_block_size_store_and_load(
|
||||
)
|
||||
)
|
||||
assert m3.load_event >= 0
|
||||
assert len(m3.load_gpu_blocks) == 1
|
||||
assert len(m3.load_cpu_blocks) == 1
|
||||
assert len(m3.load_gpu_blocks) == expected_blocks
|
||||
assert len(m3.load_cpu_blocks) == expected_blocks
|
||||
assert m3.load_gpu_blocks == [kv3.get_block_ids()[0][0]]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user