[1/N] Unify multiple-path encoder cuda graph support (#49934)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c67fe497a2
commit
e2fa28594f
@@ -65,6 +65,7 @@ from vllm.v1.sample.logits_processor import (
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
EncoderCudaGraphPathConfig,
|
||||
EncoderCudaGraphReplayBuffers,
|
||||
EncoderItemSpec,
|
||||
)
|
||||
@@ -697,9 +698,15 @@ class DeepseekOCRForCausalLM(
|
||||
modalities=["image"],
|
||||
buffer_keys=["pixel_values"],
|
||||
out_hidden_size=self.projector_config.n_embed,
|
||||
enable_dual_path_graph=True,
|
||||
global_token_per_image=self.global_image_output_token,
|
||||
local_token_per_patch=self.single_patch_output_token,
|
||||
paths={
|
||||
"global": EncoderCudaGraphPathConfig(
|
||||
min_token_budget=self.global_image_output_token
|
||||
),
|
||||
"local": EncoderCudaGraphPathConfig(
|
||||
min_token_budget=self.single_patch_output_token,
|
||||
allow_zero_tokens=True,
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
def get_encoder_cudagraph_budget_range(
|
||||
@@ -730,8 +737,10 @@ class DeepseekOCRForCausalLM(
|
||||
EncoderItemSpec(
|
||||
input_size=num_input_tokens,
|
||||
output_tokens=num_output_tokens,
|
||||
global_output_tokens=global_output_token,
|
||||
local_output_tokens=local_output_token,
|
||||
path_output_tokens={
|
||||
"global": global_output_token,
|
||||
"local": local_output_token,
|
||||
},
|
||||
)
|
||||
)
|
||||
return item_specs
|
||||
@@ -917,32 +926,34 @@ class DeepseekOCRForCausalLM(
|
||||
|
||||
def postprocess_encoder_output(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
outputs: dict[str, torch.Tensor],
|
||||
indices: list[int],
|
||||
per_item_out_tokens: list[int],
|
||||
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
|
||||
clone: bool = False,
|
||||
batch_mm_kwargs: dict[str, Any] | None = None,
|
||||
local_output: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Assemble per-image embeddings from global and local encoder outputs.
|
||||
|
||||
``output`` contains global-image features with newlines already
|
||||
``output['global']`` contains global-image features with newlines already
|
||||
inserted (from CUDA graph replay or eager fallback):
|
||||
``[B * 272, n_embed]``.
|
||||
|
||||
``local_output`` contains local-patch features without
|
||||
``output['local']`` contains local-patch features without
|
||||
newlines (from CUDA graph replay or eager fallback):
|
||||
``[P * 100, n_embed]``. May be ``None`` if no patches in batch.
|
||||
|
||||
This method:
|
||||
1. Splits ``output`` into per-image global portions.
|
||||
2. Splits ``local_output`` into per-image patch groups.
|
||||
1. Splits ``output['global']`` into per-image global portions.
|
||||
2. Splits ``output['local']`` into per-image patch groups.
|
||||
3. For each image: assembles patch grid with newlines via
|
||||
``_assemble_patch_grid``, then concatenates
|
||||
``[local_tiled, global, view_seperator]``.
|
||||
"""
|
||||
output = outputs["global"]
|
||||
local_output = outputs.get("local")
|
||||
assert batch_mm_kwargs is not None
|
||||
bsz = len(indices)
|
||||
n_embed = output.shape[-1]
|
||||
|
||||
|
||||
@@ -1696,13 +1696,12 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
|
||||
def postprocess_encoder_output(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
outputs: dict[str, torch.Tensor],
|
||||
indices: list[int],
|
||||
per_item_out_tokens: list[int],
|
||||
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
|
||||
clone: bool = False,
|
||||
batch_mm_kwargs: dict[str, Any] | None = None,
|
||||
local_output: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Post-process encoder output, directly call scatter_output_slices by default.
|
||||
@@ -1714,7 +1713,9 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
"""
|
||||
from vllm.model_executor.models.utils import scatter_output_slices
|
||||
|
||||
scatter_output_slices(output, indices, per_item_out_tokens, dest, clone)
|
||||
scatter_output_slices(
|
||||
outputs["default"], indices, per_item_out_tokens, dest, clone
|
||||
)
|
||||
|
||||
def prepare_encoder_cudagraph_capture_inputs(
|
||||
self,
|
||||
|
||||
@@ -725,6 +725,7 @@ class Step3VLForConditionalGeneration(
|
||||
def get_encoder_cudagraph_config(self):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphConfig,
|
||||
EncoderCudaGraphPathConfig,
|
||||
)
|
||||
|
||||
return EncoderCudaGraphConfig(
|
||||
@@ -734,9 +735,15 @@ class Step3VLForConditionalGeneration(
|
||||
"patch_pixel_values",
|
||||
],
|
||||
out_hidden_size=self.config.hidden_size,
|
||||
enable_dual_path_graph=True,
|
||||
global_token_per_image=self.img_output_tokens,
|
||||
local_token_per_patch=self.patch_output_tokens,
|
||||
paths={
|
||||
"global": EncoderCudaGraphPathConfig(
|
||||
min_token_budget=self.img_output_tokens
|
||||
),
|
||||
"local": EncoderCudaGraphPathConfig(
|
||||
min_token_budget=self.patch_output_tokens,
|
||||
allow_zero_tokens=True,
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
def get_encoder_cudagraph_budget_range(
|
||||
@@ -771,8 +778,10 @@ class Step3VLForConditionalGeneration(
|
||||
output_tokens=(
|
||||
self.img_output_tokens + num_patch * self.patch_output_tokens
|
||||
),
|
||||
global_output_tokens=self.img_output_tokens,
|
||||
local_output_tokens=num_patch * self.patch_output_tokens,
|
||||
path_output_tokens={
|
||||
"global": self.img_output_tokens,
|
||||
"local": num_patch * self.patch_output_tokens,
|
||||
},
|
||||
)
|
||||
for num_patch in num_patches
|
||||
]
|
||||
@@ -876,19 +885,21 @@ class Step3VLForConditionalGeneration(
|
||||
|
||||
def postprocess_encoder_output(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
outputs: dict[str, torch.Tensor],
|
||||
indices: list[int],
|
||||
per_item_out_tokens: list[int],
|
||||
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
|
||||
clone: bool = False,
|
||||
batch_mm_kwargs: dict[str, Any] | None = None,
|
||||
local_output: torch.Tensor | None = None,
|
||||
):
|
||||
"""CPU-side per-item merge after dual-path graph replay.
|
||||
|
||||
``output`` contains global-image features and ``local_output``
|
||||
``outputs['global']`` contains global-image features and ``outputs['local']``
|
||||
contains local-patch features (or ``None`` when there are no patches).
|
||||
"""
|
||||
output = outputs["global"]
|
||||
local_output = outputs.get("local")
|
||||
assert batch_mm_kwargs is not None
|
||||
num_patches = batch_mm_kwargs["num_patches"]
|
||||
hidden = output.shape[-1]
|
||||
bsz = len(indices)
|
||||
@@ -899,7 +910,7 @@ class Step3VLForConditionalGeneration(
|
||||
patch_tokens = total_patches * self.patch_output_tokens
|
||||
|
||||
global_part = output[:img_tokens].reshape(bsz, self.img_output_tokens, hidden)
|
||||
if total_patches > 0:
|
||||
if total_patches > 0 and local_output is not None:
|
||||
patch_part = local_output[:patch_tokens].reshape(
|
||||
-1, self.patch_output_tokens, hidden
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user