From cd1cb8b83c7adf1895ac05db7f7681e51b5b11b8 Mon Sep 17 00:00:00 2001 From: yhyu13 Date: Sun, 21 Jan 2024 11:12:15 +0800 Subject: [PATCH 1/2] Remove manully set use_cache; torch_dtype is not str, save model as bfloat16 used to fail; Former-commit-id: 75557fb5df16fd6eda7586cf041a16822dcfee8e --- src/llmtuner/train/tuner.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/llmtuner/train/tuner.py b/src/llmtuner/train/tuner.py index d61d9a01..2bdb9fe5 100644 --- a/src/llmtuner/train/tuner.py +++ b/src/llmtuner/train/tuner.py @@ -56,12 +56,11 @@ def export_model(args: Optional[Dict[str, Any]] = None): if not isinstance(model, PreTrainedModel): raise ValueError("The model is not a `PreTrainedModel`, export aborted.") - setattr(model.config, "use_cache", True) - if getattr(model.config, "torch_dtype", None) == "bfloat16": - model = model.to(torch.bfloat16).to("cpu") + if hasattr(model.config, "torch_dtype"): + model = model.to(getattr(model.config, "torch_dtype")).to("cpu") else: - model = model.to(torch.float16).to("cpu") - setattr(model.config, "torch_dtype", "float16") + model = model.to(torch.float32).to("cpu") + setattr(model.config, "torch_dtype", "float32") model.save_pretrained( save_directory=model_args.export_dir, From bf075c075c15f622c4ff5b0a44b28008dc70358d Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Sun, 21 Jan 2024 12:39:38 +0800 Subject: [PATCH 2/2] Update tuner.py Former-commit-id: 691420661f7115f809e76484c1f29f74637e7cd0 --- src/llmtuner/train/tuner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llmtuner/train/tuner.py b/src/llmtuner/train/tuner.py index 2bdb9fe5..c24e5eac 100644 --- a/src/llmtuner/train/tuner.py +++ b/src/llmtuner/train/tuner.py @@ -59,8 +59,8 @@ def export_model(args: Optional[Dict[str, Any]] = None): if hasattr(model.config, "torch_dtype"): model = model.to(getattr(model.config, "torch_dtype")).to("cpu") else: - model = model.to(torch.float32).to("cpu") - setattr(model.config, "torch_dtype", "float32") + model = model.to(torch.float16).to("cpu") + setattr(model.config, "torch_dtype", torch.float16) model.save_pretrained( save_directory=model_args.export_dir,