423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-15 11:20:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

support LongLoRA

Browse Source
This commit is contained in:
hiyouga
2023-09-27 21:55:50 +08:00
parent 4dd9b4d982
commit 90375f600d
8 changed files with 313 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/llmtuner/hparams/model_args.py
View File

@@ -45,7 +45,11 @@ class ModelArguments:
)
flash_attn: Optional[bool] = field(
default=False,
metadata={"help": "Enable flash attention for faster training."}
metadata={"help": "Enable FlashAttention-2 for faster training."}
)
shift_attn: Optional[bool] = field(
default=False,
metadata={"help": "Enable shift short attention (S^2-Attn) proposed by LongLoRA."}
)
checkpoint_dir: Optional[str] = field(
default=None,