[Compile] Remove torch.compile-based sequence parallelism and AsyncTP (#60517)
This commit is contained in:
@@ -116,7 +116,7 @@ from vllm.tracing import instrument
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.nvtx_pytorch_hooks import PytHooks
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
@@ -222,7 +222,6 @@ from vllm.v1.worker.ubatch_utils import (
|
||||
)
|
||||
from vllm.v1.worker.utils import (
|
||||
EncoderTimingStats,
|
||||
is_residual_scattered_for_sp,
|
||||
raise_if_nan_logits,
|
||||
)
|
||||
from vllm.v1.worker.workspace import lock_workspace
|
||||
@@ -3340,20 +3339,9 @@ class GPUModelRunner(
|
||||
) -> IntermediateTensors:
|
||||
assert self.intermediate_tensors is not None
|
||||
|
||||
tp = self.vllm_config.parallel_config.tensor_parallel_size
|
||||
is_rs = is_residual_scattered_for_sp(self.vllm_config, num_tokens)
|
||||
|
||||
# When sequence parallelism is enabled, the "residual" tensor is
|
||||
# sharded across TP ranks. All-gather it here because downstream
|
||||
# QKV + Attention needs the full residual before the SP split point.
|
||||
if sync_self:
|
||||
assert intermediate_tensors is not None
|
||||
for k, v in intermediate_tensors.items():
|
||||
is_scattered = k == "residual" and is_rs
|
||||
if is_scattered:
|
||||
local_len = num_tokens // tp
|
||||
v = get_tp_group().all_gather(v[:local_len], dim=0)
|
||||
|
||||
self.intermediate_tensors[k][:num_tokens].copy_(
|
||||
v[:num_tokens], non_blocking=True
|
||||
)
|
||||
@@ -3448,14 +3436,6 @@ class GPUModelRunner(
|
||||
async_output_copy_stream=self._get_or_create_async_output_copy_stream(),
|
||||
)
|
||||
|
||||
def _pad_for_sequence_parallelism(self, num_scheduled_tokens: int) -> int:
|
||||
# Pad tokens to multiple of tensor_parallel_size when
|
||||
# enabled collective fusion for SP
|
||||
tp_size = self.vllm_config.parallel_config.tensor_parallel_size
|
||||
if self.compilation_config.pass_config.enable_sp and tp_size > 1:
|
||||
return round_up(num_scheduled_tokens, tp_size)
|
||||
return num_scheduled_tokens
|
||||
|
||||
def _prepare_padding_mask(
|
||||
self, num_tokens_unpadded: int, num_tokens_padded: int
|
||||
) -> torch.Tensor:
|
||||
@@ -3944,8 +3924,6 @@ class GPUModelRunner(
|
||||
)
|
||||
has_lora = num_active_loras > 0 if force_has_lora is None else force_has_lora
|
||||
|
||||
num_tokens_padded = self._pad_for_sequence_parallelism(num_tokens)
|
||||
|
||||
def dispatch_cudagraph(num_tokens, disable_full=False, valid_modes=None):
|
||||
return self.cudagraph_dispatcher.dispatch(
|
||||
num_tokens=num_tokens,
|
||||
@@ -3957,18 +3935,9 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
cudagraph_mode, batch_descriptor = dispatch_cudagraph(
|
||||
num_tokens_padded, disable_full=use_cascade_attn or has_encoder_output
|
||||
num_tokens, disable_full=use_cascade_attn or has_encoder_output
|
||||
)
|
||||
num_tokens_padded = batch_descriptor.num_tokens
|
||||
if self.compilation_config.pass_config.enable_sp:
|
||||
assert (
|
||||
batch_descriptor.num_tokens
|
||||
% self.vllm_config.parallel_config.tensor_parallel_size
|
||||
== 0
|
||||
), (
|
||||
"Sequence parallelism requires num_tokens to be "
|
||||
"a multiple of tensor parallel size"
|
||||
)
|
||||
|
||||
# Extra coordination when running data-parallel since we need to coordinate
|
||||
# across ranks
|
||||
@@ -4453,15 +4422,9 @@ class GPUModelRunner(
|
||||
|
||||
sample_hidden_states = hidden_states[logits_indices]
|
||||
if not get_pp_group().is_last_rank:
|
||||
all_gather_tensors = {
|
||||
"residual": not is_residual_scattered_for_sp(
|
||||
self.vllm_config, num_tokens_padded
|
||||
)
|
||||
}
|
||||
get_pp_group().send_tensor_dict(
|
||||
hidden_states.tensors,
|
||||
all_gather_group=get_tp_group(),
|
||||
all_gather_tensors=all_gather_tensors,
|
||||
)
|
||||
logits = None
|
||||
else:
|
||||
@@ -7135,7 +7098,6 @@ class GPUModelRunner(
|
||||
min_cg_attn_backend,
|
||||
self.uniform_decode_query_len,
|
||||
use_v2_model_runner=False,
|
||||
tensor_parallel_size=self.parallel_config.tensor_parallel_size,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
is_profiling=is_profiling,
|
||||
|
||||
Reference in New Issue
Block a user