From f036b9c7bad2d2e6b6a6177e327e61045a147f2e Mon Sep 17 00:00:00 2001 From: yhyu13 Date: Sun, 21 Jan 2024 11:12:15 +0800 Subject: [PATCH 1/2] Remove manully set use_cache; torch_dtype is not str, save model as bfloat16 used to fail; Former-commit-id: 9cdbd3bfc8be3f9adc799af8db9a254a47a577a2 --- src/llmtuner/train/tuner.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/llmtuner/train/tuner.py b/src/llmtuner/train/tuner.py index d61d9a01..2bdb9fe5 100644 --- a/src/llmtuner/train/tuner.py +++ b/src/llmtuner/train/tuner.py @@ -56,12 +56,11 @@ def export_model(args: Optional[Dict[str, Any]] = None): if not isinstance(model, PreTrainedModel): raise ValueError("The model is not a `PreTrainedModel`, export aborted.") - setattr(model.config, "use_cache", True) - if getattr(model.config, "torch_dtype", None) == "bfloat16": - model = model.to(torch.bfloat16).to("cpu") + if hasattr(model.config, "torch_dtype"): + model = model.to(getattr(model.config, "torch_dtype")).to("cpu") else: - model = model.to(torch.float16).to("cpu") - setattr(model.config, "torch_dtype", "float16") + model = model.to(torch.float32).to("cpu") + setattr(model.config, "torch_dtype", "float32") model.save_pretrained( save_directory=model_args.export_dir, From 5159c9719c22f97fb1f4fa860c3fa1785f95206a Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Sun, 21 Jan 2024 12:39:38 +0800 Subject: [PATCH 2/2] Update tuner.py Former-commit-id: 662b9a9dcfadb01a903d3672e277929ec1875ed4 --- src/llmtuner/train/tuner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llmtuner/train/tuner.py b/src/llmtuner/train/tuner.py index 2bdb9fe5..c24e5eac 100644 --- a/src/llmtuner/train/tuner.py +++ b/src/llmtuner/train/tuner.py @@ -59,8 +59,8 @@ def export_model(args: Optional[Dict[str, Any]] = None): if hasattr(model.config, "torch_dtype"): model = model.to(getattr(model.config, "torch_dtype")).to("cpu") else: - model = model.to(torch.float32).to("cpu") - setattr(model.config, "torch_dtype", "float32") + model = model.to(torch.float16).to("cpu") + setattr(model.config, "torch_dtype", torch.float16) model.save_pretrained( save_directory=model_args.export_dir,