[ROCm][Bugfix] Fix chunk alignment when using context parallelism with TRITON_MLA (#46114)
Signed-off-by: Micah Williamson <micah.williamson@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Andreas Karatzas
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
1aad125815
commit
710ebaa189
@@ -13,13 +13,14 @@ import os
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Literal, NamedTuple
|
from typing import Literal, NamedTuple
|
||||||
|
|
||||||
|
import lm_eval
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from tests.evals.gsm8k.gsm8k_eval import evaluate_gsm8k
|
|
||||||
from tests.utils import RemoteOpenAIServer, create_new_process_for_each_test
|
from tests.utils import RemoteOpenAIServer, create_new_process_for_each_test
|
||||||
from vllm.config.model import RunnerOption
|
from vllm.config.model import RunnerOption
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
from ..models.registry import HF_EXAMPLE_MODELS
|
from ..models.registry import HF_EXAMPLE_MODELS
|
||||||
|
|
||||||
@@ -35,8 +36,10 @@ CP_TEST_MODELS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
# GSM8K eval configuration
|
# GSM8K eval configuration
|
||||||
NUM_QUESTIONS = 256 # Fast eval for CI
|
|
||||||
NUM_SHOTS = 5 # Few-shot examples
|
NUM_SHOTS = 5 # Few-shot examples
|
||||||
|
TASK = "gsm8k"
|
||||||
|
FILTER = "exact_match,strict-match"
|
||||||
|
NUM_CONCURRENT = 128
|
||||||
# tp accuracy with 2% buffer
|
# tp accuracy with 2% buffer
|
||||||
MIN_ACCURACY = {
|
MIN_ACCURACY = {
|
||||||
# .buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml
|
# .buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml
|
||||||
@@ -121,24 +124,34 @@ class CPTestSettings:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
CP_TEXT_GENERATION_MODELS = {
|
if current_platform.is_rocm():
|
||||||
"deepseek-ai/DeepSeek-V2-Lite-Chat": [
|
CP_TEXT_GENERATION_MODELS = {
|
||||||
CPTestSettings.detailed(dcp_multipliers=[1]),
|
"deepseek-ai/DeepSeek-V2-Lite-Chat": [
|
||||||
CPTestSettings.detailed(
|
CPTestSettings.detailed(dcp_multipliers=[1]),
|
||||||
dcp_multipliers=[0.5],
|
],
|
||||||
cp_kv_cache_interleave_size=64,
|
"Qwen/Qwen2.5-1.5B-Instruct": [
|
||||||
attn_backend="FLASHMLA",
|
CPTestSettings.detailed(dcp_multipliers=[1]),
|
||||||
),
|
],
|
||||||
],
|
}
|
||||||
"Qwen/Qwen2.5-1.5B-Instruct": [
|
else:
|
||||||
CPTestSettings.detailed(
|
CP_TEXT_GENERATION_MODELS = {
|
||||||
cp_kv_cache_interleave_size=16, attn_backend="FLASH_ATTN"
|
"deepseek-ai/DeepSeek-V2-Lite-Chat": [
|
||||||
),
|
CPTestSettings.detailed(dcp_multipliers=[1]),
|
||||||
CPTestSettings.detailed(
|
CPTestSettings.detailed(
|
||||||
cp_kv_cache_interleave_size=16, attn_backend="FLASHINFER"
|
dcp_multipliers=[0.5],
|
||||||
),
|
cp_kv_cache_interleave_size=64,
|
||||||
],
|
attn_backend="FLASHMLA",
|
||||||
}
|
),
|
||||||
|
],
|
||||||
|
"Qwen/Qwen2.5-1.5B-Instruct": [
|
||||||
|
CPTestSettings.detailed(
|
||||||
|
cp_kv_cache_interleave_size=16, attn_backend="FLASH_ATTN"
|
||||||
|
),
|
||||||
|
CPTestSettings.detailed(
|
||||||
|
cp_kv_cache_interleave_size=16, attn_backend="FLASHINFER"
|
||||||
|
),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _test_cp_gsm8k(
|
def _test_cp_gsm8k(
|
||||||
@@ -227,19 +240,23 @@ def _test_cp_gsm8k(
|
|||||||
server_args,
|
server_args,
|
||||||
max_wait_seconds=720,
|
max_wait_seconds=720,
|
||||||
) as remote_server:
|
) as remote_server:
|
||||||
host = f"http://{remote_server.host}"
|
url = f"{remote_server.url_for('v1')}/completions"
|
||||||
port = remote_server.port
|
|
||||||
|
|
||||||
# Run GSM8K evaluation
|
model_args = (
|
||||||
results = evaluate_gsm8k(
|
f"model={model_id},"
|
||||||
num_questions=NUM_QUESTIONS,
|
f"base_url={url},"
|
||||||
num_shots=NUM_SHOTS,
|
f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False"
|
||||||
host=host,
|
)
|
||||||
port=port,
|
|
||||||
|
results = lm_eval.simple_evaluate(
|
||||||
|
model="local-completions",
|
||||||
|
model_args=model_args,
|
||||||
|
tasks=TASK,
|
||||||
|
num_fewshot=NUM_SHOTS,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Validate accuracy is reasonable
|
# Validate accuracy is reasonable
|
||||||
accuracy = results["accuracy"]
|
accuracy = results["results"][TASK][FILTER]
|
||||||
min_accuracy = MIN_ACCURACY[model_id]
|
min_accuracy = MIN_ACCURACY[model_id]
|
||||||
assert accuracy >= min_accuracy, (
|
assert accuracy >= min_accuracy, (
|
||||||
f"TP+DCP accuracy too low: {accuracy:.3f} < {min_accuracy:.3f}"
|
f"TP+DCP accuracy too low: {accuracy:.3f} < {min_accuracy:.3f}"
|
||||||
|
|||||||
@@ -1532,9 +1532,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
|||||||
self.dcp_virtual_block_size = self.dcp_local_block_size * self.dcp_world_size
|
self.dcp_virtual_block_size = self.dcp_local_block_size * self.dcp_world_size
|
||||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||||
|
|
||||||
# Don't try to access the runner on AMD
|
self.page_size = self.kv_cache_spec.block_size
|
||||||
if self.aot_schedule:
|
|
||||||
self.page_size = self.kv_cache_spec.block_size
|
|
||||||
|
|
||||||
self.chunked_prefill_workspace_size = (
|
self.chunked_prefill_workspace_size = (
|
||||||
self.determine_chunked_prefill_workspace_size(vllm_config)
|
self.determine_chunked_prefill_workspace_size(vllm_config)
|
||||||
@@ -1684,12 +1682,11 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
|||||||
self.chunked_prefill_workspace_size // num_prefills_with_context_cpu
|
self.chunked_prefill_workspace_size // num_prefills_with_context_cpu
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.aot_schedule:
|
# align max_context_chunk to page_size by rounding down,
|
||||||
# align max_context_chunk to page_size by rounding down,
|
# currently the `gather_and_maybe_dequant_cache` kernel
|
||||||
# currently the `gather_and_maybe_dequant_cache` kernel
|
# cannot handle `context_chunk_starts` that are not aligned
|
||||||
# cannot handle `context_chunk_starts` that are not aligned
|
# to page_size
|
||||||
# to page_size
|
max_context_chunk = round_down(max_context_chunk, self.page_size)
|
||||||
max_context_chunk = round_down(max_context_chunk, self.page_size)
|
|
||||||
|
|
||||||
assert max_context_chunk > 0
|
assert max_context_chunk > 0
|
||||||
num_chunks = cdiv(max_context_len_cpu, max_context_chunk)
|
num_chunks = cdiv(max_context_len_cpu, max_context_chunk)
|
||||||
|
|||||||
Reference in New Issue
Block a user