[Perf][DSv4.1] Shard the Engram wkv projection across TP ranks (#58678)
Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com>
This commit is contained in:
@@ -827,7 +827,7 @@ def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
|
||||
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
engram_ops, "ReplicatedLinear", lambda *a, **k: torch.nn.Identity()
|
||||
engram_ops, "ColumnParallelLinear", lambda *a, **k: torch.nn.Identity()
|
||||
)
|
||||
stream = torch.cuda.Stream()
|
||||
kwargs = {"prefetch_stream": stream} if backend == "nvidia" else {}
|
||||
|
||||
@@ -47,7 +47,7 @@ from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.triton_utils import tl, triton
|
||||
@@ -923,13 +923,17 @@ class Engram(nn.Module):
|
||||
# suffix rule.
|
||||
self.embed_tokens = self._create_embedding(layout, layer_hash_index)
|
||||
n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads
|
||||
self.wkv = ReplicatedLinear(
|
||||
# Without sequence parallelism every TP rank holds every token, so
|
||||
# shard the output columns instead of replicating the projection.
|
||||
self.wkv = ColumnParallelLinear(
|
||||
n_hash_cols * layout.head_dim,
|
||||
self.dim * (self.hc_mult + 1),
|
||||
bias=False,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
return_bias=False,
|
||||
prefix=f"{prefix}.wkv",
|
||||
disable_tp=use_sequence_parallel,
|
||||
)
|
||||
self.q_weight = nn.Parameter(
|
||||
torch.empty(self.hc_mult, self.dim, dtype=torch.bfloat16),
|
||||
|
||||
Reference in New Issue
Block a user