DCP supports hybrid attention (#40996)
Signed-off-by: YanXu <yancey.yx@alibaba-inc.com> Signed-off-by: Jingyi Yang <girasoleyang@gmail.com> Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -153,10 +153,12 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
KVCacheSpecKind,
|
||||
KVQuantMode,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
get_kv_cache_spec_kind,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
from vllm.v1.outputs import (
|
||||
@@ -201,9 +203,11 @@ from vllm.v1.spec_decode.utils import update_num_computed_tokens_for_batch_chang
|
||||
from vllm.v1.structured_output.utils import apply_grammar_bitmask
|
||||
from vllm.v1.utils import CpuGpuBuffer, record_function_or_nullcontext
|
||||
from vllm.v1.worker import mamba_utils
|
||||
from vllm.v1.worker.block_table import SlotMappingMode
|
||||
from vllm.v1.worker.cp_utils import (
|
||||
check_attention_cp_compatibility,
|
||||
get_total_cp_world_size,
|
||||
get_dcp_dummy_context_len,
|
||||
prepare_dcp_dummy_context_metadata,
|
||||
)
|
||||
from vllm.v1.worker.dp_utils import coordinate_batch_across_dp
|
||||
from vllm.v1.worker.ec_connector_model_runner_mixin import ECConnectorModelRunnerMixin
|
||||
@@ -681,8 +685,13 @@ class GPUModelRunner(
|
||||
placeholder_block_size = (
|
||||
self.cache_config.block_size or CacheConfig.DEFAULT_BLOCK_SIZE
|
||||
)
|
||||
placeholder_max_num_blocks = cdiv(
|
||||
max(self.max_model_len, self.max_encoder_len), placeholder_block_size
|
||||
)
|
||||
self._init_block_sizes = [placeholder_block_size]
|
||||
self._init_kernel_block_sizes = [placeholder_block_size]
|
||||
self._init_max_num_blocks = [placeholder_max_num_blocks]
|
||||
self._init_slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT]
|
||||
self.input_batch = InputBatch(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
# We need to use the encoder length for encoder-decoder
|
||||
@@ -693,6 +702,7 @@ class GPUModelRunner(
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=[placeholder_block_size],
|
||||
kernel_block_sizes=[placeholder_block_size],
|
||||
max_num_blocks_per_req=[placeholder_max_num_blocks],
|
||||
num_spec_tokens=self.num_spec_tokens,
|
||||
logitsprocs=build_logitsprocs(
|
||||
self.vllm_config,
|
||||
@@ -5858,6 +5868,14 @@ class GPUModelRunner(
|
||||
num_reqs_padded = (
|
||||
batch_desc.num_reqs if batch_desc.num_reqs is not None else num_reqs
|
||||
)
|
||||
dcp_dummy_context_len = get_dcp_dummy_context_len(
|
||||
self.dcp_world_size,
|
||||
self.parallel_config.cp_kv_cache_interleave_size,
|
||||
hasattr(self, "kv_cache_config"),
|
||||
create_mixed_batch,
|
||||
is_graph_capturing,
|
||||
uniform_decode,
|
||||
)
|
||||
ubatch_slices, ubatch_slices_padded = maybe_create_ubatch_slices(
|
||||
should_ubatch,
|
||||
num_scheduled_tokens,
|
||||
@@ -5900,10 +5918,19 @@ class GPUModelRunner(
|
||||
# In the mixed batch mode (used for FI warmup), we use
|
||||
# shorter sequence lengths to run faster.
|
||||
# TODO(luka) better system for describing dummy batches
|
||||
seq_lens = torch.tensor( # type: ignore[assignment]
|
||||
[1] * num_decode_tokens + [num_prefill_tokens + 1],
|
||||
dtype=torch.int,
|
||||
)
|
||||
if dcp_dummy_context_len > 0:
|
||||
seq_lens = torch.tensor( # type: ignore[assignment]
|
||||
[1 + dcp_dummy_context_len] * num_decode_tokens
|
||||
+ [num_prefill_tokens + dcp_dummy_context_len],
|
||||
dtype=torch.int,
|
||||
)
|
||||
else:
|
||||
seq_lens = torch.tensor( # type: ignore[assignment]
|
||||
[1] * num_decode_tokens + [num_prefill_tokens + 1],
|
||||
dtype=torch.int,
|
||||
)
|
||||
elif dcp_dummy_context_len > 0:
|
||||
seq_lens = max_query_len + dcp_dummy_context_len # type: ignore[assignment]
|
||||
else:
|
||||
seq_lens = max_query_len # type: ignore[assignment]
|
||||
self.optimistic_seq_lens_cpu[:num_reqs] = seq_lens
|
||||
@@ -5919,6 +5946,17 @@ class GPUModelRunner(
|
||||
)
|
||||
self.query_start_loc.copy_to_gpu()
|
||||
|
||||
prepare_dcp_dummy_context_metadata(
|
||||
input_batch=self.input_batch,
|
||||
kv_cache_config=getattr(self, "kv_cache_config", None),
|
||||
query_pos=self.query_pos,
|
||||
positions=self.positions,
|
||||
query_start_loc=self.query_start_loc,
|
||||
num_reqs=num_reqs,
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
dcp_dummy_context_len=dcp_dummy_context_len,
|
||||
)
|
||||
|
||||
# Sync block table CPU->GPU so cleared rows from
|
||||
# remove_request() are visible to the attention metadata
|
||||
# builder. Without this, stale block IDs from finished
|
||||
@@ -7029,29 +7067,34 @@ class GPUModelRunner(
|
||||
"""
|
||||
block_sizes = []
|
||||
max_num_blocks = []
|
||||
slot_mapping_modes = []
|
||||
max_model_len = max(self.max_model_len, self.max_encoder_len)
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
||||
if isinstance(kv_cache_group.kv_cache_spec, EncoderOnlyAttentionSpec):
|
||||
kv_cache_spec = kv_cache_group.kv_cache_spec
|
||||
kv_cache_spec_kind = get_kv_cache_spec_kind(kv_cache_spec)
|
||||
if kv_cache_spec_kind == KVCacheSpecKind.ENCODER_ONLY_ATTENTION:
|
||||
continue
|
||||
block_size = kv_cache_group.kv_cache_spec.block_size
|
||||
block_size = kv_cache_spec.block_size
|
||||
block_sizes.append(block_size)
|
||||
max_num_blocks_per_req = cdiv(
|
||||
max_model_len, block_size * get_total_cp_world_size()
|
||||
if kv_cache_spec_kind == KVCacheSpecKind.MAMBA:
|
||||
slot_mapping_modes.append(SlotMappingMode.NONE)
|
||||
else:
|
||||
slot_mapping_modes.append(SlotMappingMode.TOKEN_TO_KV_SLOT)
|
||||
max_num_blocks_per_req = kv_cache_spec.max_num_blocks_per_req(
|
||||
self.vllm_config, max_model_len
|
||||
)
|
||||
if isinstance(kv_cache_group.kv_cache_spec, MambaSpec):
|
||||
max_num_blocks_per_req = (
|
||||
max_num_blocks_per_req
|
||||
if self.cache_config.enable_prefix_caching
|
||||
else 1
|
||||
) + kv_cache_group.kv_cache_spec.num_speculative_blocks
|
||||
max_num_blocks.append(max_num_blocks_per_req)
|
||||
|
||||
if (
|
||||
block_sizes != self._init_block_sizes
|
||||
or kernel_block_sizes != self._init_kernel_block_sizes
|
||||
or max_num_blocks != self._init_max_num_blocks
|
||||
or slot_mapping_modes != self._init_slot_mapping_modes
|
||||
):
|
||||
self._init_block_sizes = block_sizes
|
||||
self._init_kernel_block_sizes = kernel_block_sizes
|
||||
self._init_max_num_blocks = max_num_blocks
|
||||
self._init_slot_mapping_modes = slot_mapping_modes
|
||||
self.input_batch = InputBatch(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_model_len=max_model_len,
|
||||
@@ -7065,7 +7108,9 @@ class GPUModelRunner(
|
||||
logitsprocs=self.input_batch.logitsprocs,
|
||||
logitsprocs_need_output_token_ids=self.input_batch.logitsprocs_need_output_token_ids,
|
||||
is_pooling_model=self.is_pooling_model,
|
||||
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
|
||||
reasoning_config=self.vllm_config.reasoning_config,
|
||||
slot_mapping_modes=slot_mapping_modes,
|
||||
)
|
||||
|
||||
assert self._init_block_sizes == block_sizes, (
|
||||
|
||||
Reference in New Issue
Block a user