DCP supports hybrid attention (#40996)
Signed-off-by: YanXu <yancey.yx@alibaba-inc.com> Signed-off-by: Jingyi Yang <girasoleyang@gmail.com> Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -28,7 +28,7 @@ from vllm.v1.sample.thinking_budget_state import (
|
||||
maybe_create_thinking_budget_state_holder,
|
||||
)
|
||||
from vllm.v1.utils import copy_slice
|
||||
from vllm.v1.worker.block_table import MultiGroupBlockTable
|
||||
from vllm.v1.worker.block_table import MultiGroupBlockTable, SlotMappingMode
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -99,13 +99,14 @@ class InputBatch:
|
||||
vocab_size: int,
|
||||
block_sizes: list[int], # The block_size of each kv cache group
|
||||
kernel_block_sizes: list[int],
|
||||
max_num_blocks_per_req: list[int] | None = None,
|
||||
max_num_blocks_per_req: list[int],
|
||||
logitsprocs: LogitsProcessors | None = None,
|
||||
logitsprocs_need_output_token_ids: bool = False,
|
||||
num_spec_tokens: int = 0,
|
||||
is_pooling_model: bool = False,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
reasoning_config: ReasoningConfig | None = None,
|
||||
slot_mapping_modes: list[SlotMappingMode] | None = None,
|
||||
):
|
||||
self.thinking_budget_state_holder = maybe_create_thinking_budget_state_holder(
|
||||
reasoning_config,
|
||||
@@ -171,7 +172,6 @@ class InputBatch:
|
||||
# Block table.
|
||||
self.block_table = MultiGroupBlockTable(
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_model_len=max_model_len,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
pin_memory=PIN_MEMORY,
|
||||
device=device,
|
||||
@@ -179,6 +179,7 @@ class InputBatch:
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks=max_num_blocks_per_req,
|
||||
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
|
||||
slot_mapping_modes=slot_mapping_modes,
|
||||
)
|
||||
|
||||
# Sampling-related.
|
||||
|
||||
Reference in New Issue
Block a user