[Core] Change LoRA embedding sharding to support loading methods (#5038)

This commit is contained in:
Antoni Baum
2024-06-06 19:07:57 -07:00
committed by GitHub
parent a31cab7556
commit ccdc490dda
11 changed files with 658 additions and 126 deletions
+11 -8
View File
@@ -35,6 +35,7 @@ _BATCH_SIZE_ALIGNMENT = 8
_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
_BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
]
_NUM_WARMUP_ITERS = 2
class ModelInput(NamedTuple):
@@ -975,16 +976,18 @@ class CUDAGraphRunner:
**kwargs,
) -> None:
assert self._graph is None
# Run the model once without capturing the graph.
# Run the model a few times without capturing the graph.
# This is to make sure that the captured graph does not include the
# kernel launches for initial benchmarking (e.g., Triton autotune).
self.model(
input_ids,
positions,
kv_caches,
attn_metadata,
**kwargs,
)
# Note one iteration is not enough for torch.jit.script
for _ in range(_NUM_WARMUP_ITERS):
self.model(
input_ids,
positions,
kv_caches,
attn_metadata,
**kwargs,
)
torch.cuda.synchronize()
# Capture the graph.