move efficient_packing from data_args to model_args

This commit is contained in:
ancv
2024-07-02 18:37:55 +07:00
parent e8e6af2651
commit e8e13b0942
8 changed files with 20 additions and 18 deletions

View File

@@ -109,6 +109,12 @@ class ModelArguments:
default=False,
metadata={"help": "Enable shift short attention (S^2-Attn) proposed by LongLoRA."},
)
efficient_packing: Optional[bool] = field(
default=None,
metadata={
"help": "Whether or not to pack the sequences without cross-contamination attention for efficient training."
},
)
mixture_of_depths: Optional[Literal["convert", "load"]] = field(
default=None,
metadata={"help": "Convert the model to mixture-of-depths (MoD) or load the MoD model."},