423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-23 22:32:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

support train from scratch #4033 #4075

Browse Source 
Former-commit-id: a12a506c3d2ba85975a5990c46d2e055cdfe0f2e
This commit is contained in:
hiyouga 2024-06-06 02:43:19 +08:00
parent cafbb79d3a
commit 3fcb678d00
2 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llamafactory/hparams/model_args.py
View File

@ -101,6 +101,10 @@ class ModelArguments:
default=False, default=False,
metadata={"help": "Whether or not to upcast the output of lm_head in fp32."}, metadata={"help": "Whether or not to upcast the output of lm_head in fp32."},
) )
train_from_scratch: bool = field(
default=False,
metadata={"help": "Whether or not to randomly initialize the model weights."},
)
infer_backend: Literal["huggingface", "vllm"] = field( infer_backend: Literal["huggingface", "vllm"] = field(
default="huggingface", default="huggingface",
metadata={"help": "Backend engine used at inference."}, metadata={"help": "Backend engine used at inference."},

2
src/llamafactory/model/loader.py
View File

@ -131,6 +131,8 @@ def load_model(
model = load_mod_pretrained_model(**init_kwargs) model = load_mod_pretrained_model(**init_kwargs)
elif model_args.visual_inputs: elif model_args.visual_inputs:
model = AutoModelForVision2Seq.from_pretrained(**init_kwargs) model = AutoModelForVision2Seq.from_pretrained(**init_kwargs)
elif model_args.train_from_scratch:
model = AutoModelForCausalLM.from_config(config)
else: else:
model = AutoModelForCausalLM.from_pretrained(**init_kwargs) model = AutoModelForCausalLM.from_pretrained(**init_kwargs)