DCP supports hybrid attention (#40996)
Signed-off-by: YanXu <yancey.yx@alibaba-inc.com> Signed-off-by: Jingyi Yang <girasoleyang@gmail.com> Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -33,6 +33,7 @@ CP_TEST_MODELS = [
|
||||
# [LANGUAGE GENERATION]
|
||||
"deepseek-ai/DeepSeek-V2-Lite-Chat",
|
||||
"Qwen/Qwen2.5-1.5B-Instruct",
|
||||
"Qwen/Qwen3.5-0.8B", # hybrid attention model
|
||||
]
|
||||
|
||||
# GSM8K eval configuration
|
||||
@@ -46,6 +47,7 @@ MIN_ACCURACY = {
|
||||
"deepseek-ai/DeepSeek-V2-Lite-Chat": 0.64,
|
||||
# .buildkite/lm-eval-harness/configs/Qwen2.5-1.5B-Instruct.yaml
|
||||
"Qwen/Qwen2.5-1.5B-Instruct": 0.52,
|
||||
"Qwen/Qwen3.5-0.8B": 0.33,
|
||||
}
|
||||
|
||||
|
||||
@@ -151,6 +153,12 @@ else:
|
||||
cp_kv_cache_interleave_size=16, attn_backend="FLASHINFER"
|
||||
),
|
||||
],
|
||||
"Qwen/Qwen3.5-0.8B": [
|
||||
CPTestSettings.detailed(
|
||||
cp_kv_cache_interleave_size=16,
|
||||
attn_backend="FLASH_ATTN",
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
get_tp_group,
|
||||
get_world_group,
|
||||
graph_capture,
|
||||
init_distributed_environment,
|
||||
@@ -199,6 +200,52 @@ def test_pynccl_all_gather():
|
||||
distributed_run(all_gather_worker_fn, 2)
|
||||
|
||||
|
||||
@worker_fn_wrapper
|
||||
def cuda_communicator_all_gather_dim_worker_fn():
|
||||
with ensure_current_vllm_config():
|
||||
ensure_model_parallel_initialized(2, 1)
|
||||
|
||||
tp_group = get_tp_group()
|
||||
comm = tp_group.device_communicator
|
||||
assert comm is not None
|
||||
|
||||
rank = tp_group.rank_in_group
|
||||
world_size = tp_group.world_size
|
||||
device = tp_group.device
|
||||
|
||||
shape = (2, 3, 4)
|
||||
num_elems = 1
|
||||
for size in shape:
|
||||
num_elems *= size
|
||||
|
||||
for dim in (1, -1):
|
||||
tensor = (
|
||||
torch.arange(num_elems, dtype=torch.float32, device=device).reshape(shape)
|
||||
+ rank * num_elems
|
||||
)
|
||||
expected = torch.cat(
|
||||
[
|
||||
torch.arange(num_elems, dtype=torch.float32, device=device).reshape(
|
||||
shape
|
||||
)
|
||||
+ r * num_elems
|
||||
for r in range(world_size)
|
||||
],
|
||||
dim=dim,
|
||||
)
|
||||
|
||||
result = comm.all_gather(tensor, dim=dim)
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 2, reason="Need at least 2 GPUs to run the test."
|
||||
)
|
||||
def test_cuda_communicator_all_gather_dim_not_zero():
|
||||
distributed_run(cuda_communicator_all_gather_dim_worker_fn, 2)
|
||||
|
||||
|
||||
@worker_fn_wrapper
|
||||
def all_gatherv_worker_fn():
|
||||
pynccl_comm = PyNcclCommunicator(
|
||||
|
||||
@@ -43,6 +43,11 @@ HYBRID_MODELS = [
|
||||
"tiny-random/qwen3-next-moe",
|
||||
]
|
||||
|
||||
HYBRID_MODELS_REQUIRING_CHUNKED_PREFILL = {
|
||||
"LiquidAI/LFM2-1.2B",
|
||||
"tiny-random/qwen3-next-moe",
|
||||
}
|
||||
|
||||
FULL_CUDA_GRAPH_MODELS = [
|
||||
"ai21labs/Jamba-tiny-dev",
|
||||
"pfnet/plamo-2-1b",
|
||||
@@ -92,8 +97,15 @@ def test_models(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
extra_kwargs = {}
|
||||
if model in HYBRID_MODELS_REQUIRING_CHUNKED_PREFILL:
|
||||
extra_kwargs["enable_chunked_prefill"] = True
|
||||
|
||||
with vllm_runner(
|
||||
model, max_num_seqs=MAX_NUM_SEQS, attention_backend=ATTN_BACKEND
|
||||
model,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
attention_backend=ATTN_BACKEND,
|
||||
**extra_kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
|
||||
@@ -171,6 +171,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
"Describe this video in one sentence."
|
||||
),
|
||||
needs_video_metadata=True,
|
||||
vllm_runner_kwargs={"enable_chunked_prefill": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"internvl": VitCudagraphTestConfig(
|
||||
|
||||
@@ -1121,14 +1121,14 @@ def test_is_chunked_prefill_supported(
|
||||
(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
"hybrid",
|
||||
False,
|
||||
"Hybrid models do not support prefix caching since the feature is still experimental.", # noqa: E501
|
||||
True,
|
||||
"Generative hybrid models support prefix caching.", # noqa: E501
|
||||
),
|
||||
(
|
||||
"ibm-granite/granite-4.0-h-small",
|
||||
"hybrid",
|
||||
False,
|
||||
"Hybrid models do not support prefix caching since the feature is still experimental.", # noqa: E501
|
||||
True,
|
||||
"Generative hybrid models support prefix caching.", # noqa: E501
|
||||
),
|
||||
(
|
||||
"state-spaces/mamba-130m-hf",
|
||||
|
||||
@@ -38,6 +38,7 @@ def mock_model_runner_with_input_batch():
|
||||
vocab_size=32000,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
logitsprocs=None,
|
||||
is_pooling_model=False,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
|
||||
from vllm.v1.worker.cp_utils import should_skip_dcp_context_attention
|
||||
|
||||
|
||||
def test_skip_gate_only_for_zero_context():
|
||||
assert should_skip_dcp_context_attention(torch.zeros(3, dtype=torch.int32))
|
||||
assert not should_skip_dcp_context_attention(
|
||||
torch.tensor([0, 5, 0], dtype=torch.int32)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dcp_world_size,interleave_size,context_len",
|
||||
[(2, 16, 10), (4, 16, 10), (8, 16, 10), (4, 1, 2)],
|
||||
)
|
||||
def test_skip_gate_rank_invariant_with_divergent_local_context(
|
||||
dcp_world_size: int, interleave_size: int, context_len: int
|
||||
):
|
||||
"""Contexts shorter than a full interleave round land entirely on a
|
||||
subset of DCP ranks, so the per-rank local context lengths diverge:
|
||||
some ranks hold zero local context while others hold all of it. Ranks
|
||||
with zero local context must still take the collective (non-skip) path,
|
||||
otherwise the query all-gather in _forward_with_dcp deadlocks across
|
||||
ranks. The skip gate must therefore depend only on the rank-invariant
|
||||
global context lengths, never on get_dcp_local_seq_lens output.
|
||||
"""
|
||||
context_kv_lens = torch.tensor([context_len], dtype=torch.int32)
|
||||
local_maxes = [
|
||||
int(
|
||||
get_dcp_local_seq_lens(
|
||||
context_kv_lens, dcp_world_size, rank, interleave_size
|
||||
).max()
|
||||
)
|
||||
for rank in range(dcp_world_size)
|
||||
]
|
||||
# Precondition: the local view diverges across ranks.
|
||||
assert 0 in local_maxes
|
||||
assert max(local_maxes) > 0
|
||||
# The batch still has context globally, so no rank may skip.
|
||||
assert not should_skip_dcp_context_attention(context_kv_lens)
|
||||
@@ -238,6 +238,7 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
reqs: list[CachedRequestState] = []
|
||||
req_id_reqs = {}
|
||||
@@ -332,6 +333,7 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
ref_input_batch: InputBatch = InputBatch(
|
||||
max_num_reqs=batch_size,
|
||||
@@ -341,6 +343,7 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
reqs: list[CachedRequestState] = []
|
||||
@@ -409,6 +412,7 @@ def test_pooling_prompt_lens_not_aliased(device: str):
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=True,
|
||||
)
|
||||
|
||||
@@ -444,6 +448,7 @@ def test_placeholder_spec_token_ids_written_verbatim():
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[1],
|
||||
)
|
||||
req = CachedRequestState(
|
||||
req_id="req",
|
||||
@@ -491,6 +496,7 @@ def test_pooling_metadata_token_id_buffers(
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=True,
|
||||
)
|
||||
req = _construct_pooling_request(0, PoolingParams(**pooling_params))
|
||||
|
||||
@@ -89,6 +89,7 @@ def initialize_kv_cache(runner: GPUModelRunner):
|
||||
kernel_block_sizes=[
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
|
||||
],
|
||||
max_num_blocks_per_req=[NUM_BLOCKS],
|
||||
)
|
||||
runner.initialize_attn_backend(kv_cache_config)
|
||||
|
||||
@@ -1397,6 +1398,7 @@ def test_input_batch_with_kernel_block_sizes():
|
||||
vocab_size=vocab_size,
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks_per_req=[16, 8],
|
||||
)
|
||||
|
||||
# Verify that block tables were created with kernel block sizes
|
||||
@@ -1457,6 +1459,7 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init):
|
||||
vocab_size=runner.model_config.get_vocab_size(),
|
||||
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[NUM_BLOCKS],
|
||||
) # Use kernel block size
|
||||
|
||||
runner.initialize_attn_backend(kv_cache_config)
|
||||
|
||||
Reference in New Issue
Block a user