423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-28 01:30:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

support LongLoRA

Browse Source 
Former-commit-id: 0832ed37e7947d699f17375648a52f80752c2b6b
This commit is contained in:
hiyouga
2023-09-27 21:55:50 +08:00
parent 73c48d0463
commit 20130b486c
8 changed files with 313 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/llmtuner/hparams/model_args.py
View File

@@ -45,7 +45,11 @@ class ModelArguments:
)
flash_attn: Optional[bool] = field(
default=False,
metadata={"help": "Enable flash attention for faster training."}
metadata={"help": "Enable FlashAttention-2 for faster training."}
)
shift_attn: Optional[bool] = field(
default=False,
metadata={"help": "Enable shift short attention (S^2-Attn) proposed by LongLoRA."}
)
checkpoint_dir: Optional[str] = field(
default=None,