[PCP][Spec Decode] Adds PCP support for single-module MTP and replicated DSpark. (#56107)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Summer Yang
OpenAI Codex
OpenAI Codex
parent
e7edf17cea
commit
980c16c8e4
@@ -48,6 +48,7 @@ def _make_fake_speculator(
|
||||
)
|
||||
return SimpleNamespace(
|
||||
arange=torch.arange(max_num_reqs + 1, dtype=torch.int32, device="cpu"),
|
||||
draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool),
|
||||
block_tables=fake_block_tables,
|
||||
input_buffers=fake_input_buffers,
|
||||
attn_groups=[],
|
||||
|
||||
@@ -94,6 +94,7 @@ def _make_speculator(
|
||||
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.pcp_manager = None
|
||||
speculator.vllm_config = None
|
||||
speculator.input_buffers = SimpleNamespace(
|
||||
input_ids=torch.arange(4),
|
||||
|
||||
@@ -11,7 +11,6 @@ from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
def _copy_to_cpu(value, out=None, device=None):
|
||||
@@ -125,6 +124,7 @@ def _make_global_decode_batch(
|
||||
|
||||
base = InputBatch.make_dummy(num_reqs, num_tokens, buffers)
|
||||
buffers.seq_lens[:num_reqs] = torch.from_numpy(seq_lens_np).to(device)
|
||||
buffers.positions[:num_reqs] = torch.tensor(num_computed_tokens, device=device)
|
||||
query_start_loc_np = np.arange(num_reqs + 1, dtype=np.int32)
|
||||
buffers.query_start_loc[: num_reqs + 1] = torch.from_numpy(query_start_loc_np).to(
|
||||
device
|
||||
@@ -166,19 +166,10 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch():
|
||||
the field afterwards from the PCP-owned buffers.
|
||||
"""
|
||||
device = torch.device("cuda:0")
|
||||
req_states = RequestState(
|
||||
max_num_reqs=4,
|
||||
max_model_len=64,
|
||||
max_num_batched_tokens=8,
|
||||
num_speculative_steps=1,
|
||||
vocab_size=8,
|
||||
device=device,
|
||||
)
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=device,
|
||||
req_states=req_states,
|
||||
max_num_reqs=4,
|
||||
max_num_tokens=8,
|
||||
dcp_world_size=2,
|
||||
|
||||
Reference in New Issue
Block a user