[V1] EP/TP MoE + DP Attention (#13931)
This commit is contained in:
+15
-7
@@ -25,16 +25,22 @@ batchsize_logging_interval: float = envs.VLLM_LOG_BATCHSIZE_INTERVAL
|
||||
batchsize_forward_time: defaultdict = defaultdict(list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DPMetadata:
|
||||
num_tokens_across_dp: list[int]
|
||||
cu_tokens_across_dp_cpu: torch.Tensor
|
||||
|
||||
|
||||
@dataclass
|
||||
class ForwardContext:
|
||||
# copy from vllm_config.compilation_config.static_forward_context
|
||||
attn_layers: dict[str, Any]
|
||||
no_compile_layers: dict[str, Any]
|
||||
# TODO: extend to support per-layer dynamic forward context
|
||||
attn_metadata: "AttentionMetadata" # set dynamically for each forward pass
|
||||
# TODO: remove after making all virtual_engines share the same kv cache
|
||||
virtual_engine: int # set dynamically for each forward pass
|
||||
num_tokens_across_dp: Optional[
|
||||
list[int]] = None # set dynamically for each forward pass
|
||||
# set dynamically for each forward pass
|
||||
dp_metadata: Optional[DPMetadata] = None
|
||||
|
||||
|
||||
_forward_context: Optional[ForwardContext] = None
|
||||
@@ -61,7 +67,7 @@ def set_forward_context(attn_metadata: Any,
|
||||
need_to_track_batchsize = track_batchsize and attn_metadata is not None
|
||||
if need_to_track_batchsize:
|
||||
forward_start_time = time.perf_counter()
|
||||
num_tokens_across_dp = None
|
||||
dp_metadata: Optional[DPMetadata] = None
|
||||
if vllm_config.parallel_config.data_parallel_size > 1:
|
||||
dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
@@ -82,15 +88,17 @@ def set_forward_context(attn_metadata: Any,
|
||||
dtype=torch.int32)
|
||||
from vllm.distributed.parallel_state import get_dp_group
|
||||
dist.all_reduce(num_tokens_tensor, group=get_dp_group().cpu_group)
|
||||
num_tokens_across_dp = num_tokens_tensor.tolist()
|
||||
cu_tokens_across_dp_cpu = torch.cumsum(num_tokens_tensor, dim=0)
|
||||
dp_metadata = DPMetadata(num_tokens_across_dp, cu_tokens_across_dp_cpu)
|
||||
|
||||
global _forward_context
|
||||
prev_context = _forward_context
|
||||
_forward_context = ForwardContext(
|
||||
attn_layers=vllm_config.compilation_config.static_forward_context,
|
||||
no_compile_layers=vllm_config.compilation_config.
|
||||
static_forward_context,
|
||||
virtual_engine=virtual_engine,
|
||||
attn_metadata=attn_metadata,
|
||||
num_tokens_across_dp=num_tokens_across_dp)
|
||||
dp_metadata=dp_metadata)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user