[1/N] Unify multiple-path encoder cuda graph support (#49934)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Isotr0py
2026-08-02 10:33:23 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent c67fe497a2
commit e2fa28594f
7 changed files with 193 additions and 388 deletions
+22 -11
View File
@@ -65,6 +65,7 @@ from vllm.v1.sample.logits_processor import (
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
EncoderCudaGraphConfig,
EncoderCudaGraphPathConfig,
EncoderCudaGraphReplayBuffers,
EncoderItemSpec,
)
@@ -697,9 +698,15 @@ class DeepseekOCRForCausalLM(
modalities=["image"],
buffer_keys=["pixel_values"],
out_hidden_size=self.projector_config.n_embed,
enable_dual_path_graph=True,
global_token_per_image=self.global_image_output_token,
local_token_per_patch=self.single_patch_output_token,
paths={
"global": EncoderCudaGraphPathConfig(
min_token_budget=self.global_image_output_token
),
"local": EncoderCudaGraphPathConfig(
min_token_budget=self.single_patch_output_token,
allow_zero_tokens=True,
),
},
)
def get_encoder_cudagraph_budget_range(
@@ -730,8 +737,10 @@ class DeepseekOCRForCausalLM(
EncoderItemSpec(
input_size=num_input_tokens,
output_tokens=num_output_tokens,
global_output_tokens=global_output_token,
local_output_tokens=local_output_token,
path_output_tokens={
"global": global_output_token,
"local": local_output_token,
},
)
)
return item_specs
@@ -917,32 +926,34 @@ class DeepseekOCRForCausalLM(
def postprocess_encoder_output(
self,
output: torch.Tensor,
outputs: dict[str, torch.Tensor],
indices: list[int],
per_item_out_tokens: list[int],
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
clone: bool = False,
batch_mm_kwargs: dict[str, Any] | None = None,
local_output: torch.Tensor | None = None,
) -> None:
"""
Assemble per-image embeddings from global and local encoder outputs.
``output`` contains global-image features with newlines already
``output['global']`` contains global-image features with newlines already
inserted (from CUDA graph replay or eager fallback):
``[B * 272, n_embed]``.
``local_output`` contains local-patch features without
``output['local']`` contains local-patch features without
newlines (from CUDA graph replay or eager fallback):
``[P * 100, n_embed]``. May be ``None`` if no patches in batch.
This method:
1. Splits ``output`` into per-image global portions.
2. Splits ``local_output`` into per-image patch groups.
1. Splits ``output['global']`` into per-image global portions.
2. Splits ``output['local']`` into per-image patch groups.
3. For each image: assembles patch grid with newlines via
``_assemble_patch_grid``, then concatenates
``[local_tiled, global, view_seperator]``.
"""
output = outputs["global"]
local_output = outputs.get("local")
assert batch_mm_kwargs is not None
bsz = len(indices)
n_embed = output.shape[-1]