[ROCm][Bugfix] Fix chunk alignment when using context parallelism with TRITON_MLA (#46114)

Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Micah Williamson
2026-06-25 00:07:28 -04:00
committed by GitHub
co-authored by Andreas Karatzas mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1aad125815
commit 710ebaa189
2 changed files with 52 additions and 38 deletions
+46 -29
View File
@@ -13,13 +13,14 @@ import os
from dataclasses import dataclass from dataclasses import dataclass
from typing import Literal, NamedTuple from typing import Literal, NamedTuple
import lm_eval
import pytest import pytest
import torch import torch
from tests.evals.gsm8k.gsm8k_eval import evaluate_gsm8k
from tests.utils import RemoteOpenAIServer, create_new_process_for_each_test from tests.utils import RemoteOpenAIServer, create_new_process_for_each_test
from vllm.config.model import RunnerOption from vllm.config.model import RunnerOption
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.platforms import current_platform
from ..models.registry import HF_EXAMPLE_MODELS from ..models.registry import HF_EXAMPLE_MODELS
@@ -35,8 +36,10 @@ CP_TEST_MODELS = [
] ]
# GSM8K eval configuration # GSM8K eval configuration
NUM_QUESTIONS = 256 # Fast eval for CI
NUM_SHOTS = 5 # Few-shot examples NUM_SHOTS = 5 # Few-shot examples
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
NUM_CONCURRENT = 128
# tp accuracy with 2% buffer # tp accuracy with 2% buffer
MIN_ACCURACY = { MIN_ACCURACY = {
# .buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml # .buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml
@@ -121,24 +124,34 @@ class CPTestSettings:
) )
CP_TEXT_GENERATION_MODELS = { if current_platform.is_rocm():
"deepseek-ai/DeepSeek-V2-Lite-Chat": [ CP_TEXT_GENERATION_MODELS = {
CPTestSettings.detailed(dcp_multipliers=[1]), "deepseek-ai/DeepSeek-V2-Lite-Chat": [
CPTestSettings.detailed( CPTestSettings.detailed(dcp_multipliers=[1]),
dcp_multipliers=[0.5], ],
cp_kv_cache_interleave_size=64, "Qwen/Qwen2.5-1.5B-Instruct": [
attn_backend="FLASHMLA", CPTestSettings.detailed(dcp_multipliers=[1]),
), ],
], }
"Qwen/Qwen2.5-1.5B-Instruct": [ else:
CPTestSettings.detailed( CP_TEXT_GENERATION_MODELS = {
cp_kv_cache_interleave_size=16, attn_backend="FLASH_ATTN" "deepseek-ai/DeepSeek-V2-Lite-Chat": [
), CPTestSettings.detailed(dcp_multipliers=[1]),
CPTestSettings.detailed( CPTestSettings.detailed(
cp_kv_cache_interleave_size=16, attn_backend="FLASHINFER" dcp_multipliers=[0.5],
), cp_kv_cache_interleave_size=64,
], attn_backend="FLASHMLA",
} ),
],
"Qwen/Qwen2.5-1.5B-Instruct": [
CPTestSettings.detailed(
cp_kv_cache_interleave_size=16, attn_backend="FLASH_ATTN"
),
CPTestSettings.detailed(
cp_kv_cache_interleave_size=16, attn_backend="FLASHINFER"
),
],
}
def _test_cp_gsm8k( def _test_cp_gsm8k(
@@ -227,19 +240,23 @@ def _test_cp_gsm8k(
server_args, server_args,
max_wait_seconds=720, max_wait_seconds=720,
) as remote_server: ) as remote_server:
host = f"http://{remote_server.host}" url = f"{remote_server.url_for('v1')}/completions"
port = remote_server.port
# Run GSM8K evaluation model_args = (
results = evaluate_gsm8k( f"model={model_id},"
num_questions=NUM_QUESTIONS, f"base_url={url},"
num_shots=NUM_SHOTS, f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False"
host=host, )
port=port,
results = lm_eval.simple_evaluate(
model="local-completions",
model_args=model_args,
tasks=TASK,
num_fewshot=NUM_SHOTS,
) )
# Validate accuracy is reasonable # Validate accuracy is reasonable
accuracy = results["accuracy"] accuracy = results["results"][TASK][FILTER]
min_accuracy = MIN_ACCURACY[model_id] min_accuracy = MIN_ACCURACY[model_id]
assert accuracy >= min_accuracy, ( assert accuracy >= min_accuracy, (
f"TP+DCP accuracy too low: {accuracy:.3f} < {min_accuracy:.3f}" f"TP+DCP accuracy too low: {accuracy:.3f} < {min_accuracy:.3f}"
@@ -1532,9 +1532,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.dcp_virtual_block_size = self.dcp_local_block_size * self.dcp_world_size self.dcp_virtual_block_size = self.dcp_local_block_size * self.dcp_world_size
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
# Don't try to access the runner on AMD self.page_size = self.kv_cache_spec.block_size
if self.aot_schedule:
self.page_size = self.kv_cache_spec.block_size
self.chunked_prefill_workspace_size = ( self.chunked_prefill_workspace_size = (
self.determine_chunked_prefill_workspace_size(vllm_config) self.determine_chunked_prefill_workspace_size(vllm_config)
@@ -1684,12 +1682,11 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.chunked_prefill_workspace_size // num_prefills_with_context_cpu self.chunked_prefill_workspace_size // num_prefills_with_context_cpu
) )
if self.aot_schedule: # align max_context_chunk to page_size by rounding down,
# align max_context_chunk to page_size by rounding down, # currently the `gather_and_maybe_dequant_cache` kernel
# currently the `gather_and_maybe_dequant_cache` kernel # cannot handle `context_chunk_starts` that are not aligned
# cannot handle `context_chunk_starts` that are not aligned # to page_size
# to page_size max_context_chunk = round_down(max_context_chunk, self.page_size)
max_context_chunk = round_down(max_context_chunk, self.page_size)
assert max_context_chunk > 0 assert max_context_chunk > 0
num_chunks = cdiv(max_context_len_cpu, max_context_chunk) num_chunks = cdiv(max_context_len_cpu, max_context_chunk)