423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-03 20:22:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix recursion error

Browse Source 
Former-commit-id: b79ca8781e368b17c83e585f1644346628d7e2de
This commit is contained in:
hiyouga 2023-10-22 16:28:37 +08:00
parent d6c77d9196
commit ce112dc744
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/tuner/core/utils.py
View File

@ -54,7 +54,7 @@ def prepare_model_for_training(
input_embed: torch.nn.Embedding = model.get_input_embeddings() input_embed: torch.nn.Embedding = model.get_input_embeddings()
def noisy_forward(self: torch.nn.Embedding, x: torch.Tensor) -> torch.Tensor: def noisy_forward(self: torch.nn.Embedding, x: torch.Tensor) -> torch.Tensor:
embeddings = input_embed.forward(x) embeddings = torch.nn.Embedding.forward(self, x)
if self.training: if self.training:
dims = self.num_embeddings * self.embedding_dim dims = self.num_embeddings * self.embedding_dim
mag_norm = finetuning_args.neft_alpha / (dims ** 0.5) mag_norm = finetuning_args.neft_alpha / (dims ** 0.5)
@ -79,7 +79,7 @@ def prepare_model_for_training(
input_dtype = output_layer.weight.dtype input_dtype = output_layer.weight.dtype
def forward_in_fp32(self, x: torch.Tensor) -> torch.Tensor: def forward_in_fp32(self, x: torch.Tensor) -> torch.Tensor:
return output_layer.forward(x.to(input_dtype)).to(torch.float32) return torch.nn.Linear.forward(self, x.to(input_dtype)).to(torch.float32)
output_layer.forward = MethodType(forward_in_fp32, output_layer) output_layer.forward = MethodType(forward_in_fp32, output_layer)