DCP supports hybrid attention (#40996)
Signed-off-by: YanXu <yancey.yx@alibaba-inc.com> Signed-off-by: Jingyi Yang <girasoleyang@gmail.com> Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -33,6 +33,7 @@ CP_TEST_MODELS = [
|
||||
# [LANGUAGE GENERATION]
|
||||
"deepseek-ai/DeepSeek-V2-Lite-Chat",
|
||||
"Qwen/Qwen2.5-1.5B-Instruct",
|
||||
"Qwen/Qwen3.5-0.8B", # hybrid attention model
|
||||
]
|
||||
|
||||
# GSM8K eval configuration
|
||||
@@ -46,6 +47,7 @@ MIN_ACCURACY = {
|
||||
"deepseek-ai/DeepSeek-V2-Lite-Chat": 0.64,
|
||||
# .buildkite/lm-eval-harness/configs/Qwen2.5-1.5B-Instruct.yaml
|
||||
"Qwen/Qwen2.5-1.5B-Instruct": 0.52,
|
||||
"Qwen/Qwen3.5-0.8B": 0.33,
|
||||
}
|
||||
|
||||
|
||||
@@ -151,6 +153,12 @@ else:
|
||||
cp_kv_cache_interleave_size=16, attn_backend="FLASHINFER"
|
||||
),
|
||||
],
|
||||
"Qwen/Qwen3.5-0.8B": [
|
||||
CPTestSettings.detailed(
|
||||
cp_kv_cache_interleave_size=16,
|
||||
attn_backend="FLASH_ATTN",
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
get_tp_group,
|
||||
get_world_group,
|
||||
graph_capture,
|
||||
init_distributed_environment,
|
||||
@@ -199,6 +200,52 @@ def test_pynccl_all_gather():
|
||||
distributed_run(all_gather_worker_fn, 2)
|
||||
|
||||
|
||||
@worker_fn_wrapper
|
||||
def cuda_communicator_all_gather_dim_worker_fn():
|
||||
with ensure_current_vllm_config():
|
||||
ensure_model_parallel_initialized(2, 1)
|
||||
|
||||
tp_group = get_tp_group()
|
||||
comm = tp_group.device_communicator
|
||||
assert comm is not None
|
||||
|
||||
rank = tp_group.rank_in_group
|
||||
world_size = tp_group.world_size
|
||||
device = tp_group.device
|
||||
|
||||
shape = (2, 3, 4)
|
||||
num_elems = 1
|
||||
for size in shape:
|
||||
num_elems *= size
|
||||
|
||||
for dim in (1, -1):
|
||||
tensor = (
|
||||
torch.arange(num_elems, dtype=torch.float32, device=device).reshape(shape)
|
||||
+ rank * num_elems
|
||||
)
|
||||
expected = torch.cat(
|
||||
[
|
||||
torch.arange(num_elems, dtype=torch.float32, device=device).reshape(
|
||||
shape
|
||||
)
|
||||
+ r * num_elems
|
||||
for r in range(world_size)
|
||||
],
|
||||
dim=dim,
|
||||
)
|
||||
|
||||
result = comm.all_gather(tensor, dim=dim)
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 2, reason="Need at least 2 GPUs to run the test."
|
||||
)
|
||||
def test_cuda_communicator_all_gather_dim_not_zero():
|
||||
distributed_run(cuda_communicator_all_gather_dim_worker_fn, 2)
|
||||
|
||||
|
||||
@worker_fn_wrapper
|
||||
def all_gatherv_worker_fn():
|
||||
pynccl_comm = PyNcclCommunicator(
|
||||
|
||||
Reference in New Issue
Block a user