[Kernel] LoRA - Enable CUDAGraphs for V1 (#14626)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
parent
32ef4983cd
commit
0b1cfa6180
+9
-6
@@ -237,16 +237,19 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
self.embeddings_weights[:embeddings.shape[0]].copy_(embeddings)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
added_tokens_mask = x > self.base_layer.org_vocab_size - 1
|
||||
embeddings_indices = self.punica_wrapper.embeddings_indices
|
||||
indices = embeddings_indices[1].view_as(x)
|
||||
added_tokens_mask = torch.where(x > self.base_layer.org_vocab_size - 1,
|
||||
1, 0)
|
||||
embeddings_indices = torch.narrow(
|
||||
self.punica_wrapper._embeddings_indices, 1, 0, x.size(0))
|
||||
|
||||
indices = embeddings_indices[1]
|
||||
full_lora_a_embeddings = F.embedding(
|
||||
x + indices,
|
||||
self.lora_a_stacked_2d,
|
||||
)
|
||||
indices = embeddings_indices[0].view_as(x)
|
||||
full_output = self.base_layer.forward(
|
||||
x.add_(indices * added_tokens_mask))
|
||||
indices = embeddings_indices[0]
|
||||
full_output = self.base_layer.forward(x +
|
||||
(indices * added_tokens_mask))
|
||||
|
||||
full_output_org = full_output
|
||||
if full_output.ndim == 3:
|
||||
|
||||
@@ -254,7 +254,9 @@ class PunicaWrapperGPU(PunicaWrapperBase, V1KernelMixin):
|
||||
y_org = y
|
||||
y = y.view(-1, y.shape[-1])
|
||||
if lora_bias_stacked is not None:
|
||||
self._apply_bias(self.token_lora_indices, y, output_slices,
|
||||
token_lora_indices = torch.narrow(self._token_lora_indices, 0, 0,
|
||||
y.size(0))
|
||||
self._apply_bias(token_lora_indices, y, output_slices,
|
||||
lora_bias_stacked)
|
||||
|
||||
if env.VLLM_USE_V1:
|
||||
@@ -365,7 +367,9 @@ class PunicaWrapperGPU(PunicaWrapperBase, V1KernelMixin):
|
||||
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
|
||||
if lora_bias_stacked is not None:
|
||||
assert len(lora_bias_stacked) == len(output_slices)
|
||||
y = self._apply_bias(self.token_lora_indices, y, output_slices,
|
||||
token_lora_indices = torch.narrow(self._token_lora_indices, 0, 0,
|
||||
y.size(0))
|
||||
y = self._apply_bias(token_lora_indices, y, output_slices,
|
||||
lora_bias_stacked)
|
||||
|
||||
if buffer is None:
|
||||
|
||||
Reference in New Issue
Block a user