423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-17 04:10:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

reimplement neftune

Browse Source
This commit is contained in:
hiyouga
2023-10-22 16:15:08 +08:00
parent b42a145253
commit 7b4acf7265
9 changed files with 36 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/llmtuner/hparams/finetuning_args.py
View File

@@ -75,9 +75,13 @@ class FinetuningArguments:
default=0.1,
metadata={"help": "The beta parameter for the DPO loss."}
)
neftune_noise_alpha: Optional[float] = field(
default=None,
metadata={"help": "The alpha parameter for the NEFTune noise. By setting this the NEFTune optimization will be activated."}
upcast_layernorm: Optional[bool] = field(
default=False,
metadata={"help": "Whether to upcast the layernorm weights in fp32."}
)
neft_alpha: Optional[float] = field(
default=0,
metadata={"help": "The alpha parameter to control the noise magnitude in NEFTune."}
)
def __post_init__(self):