diff --git a/src/llmtuner/model/loader.py b/src/llmtuner/model/loader.py
index 18a33043..14ea7095 100644
--- a/src/llmtuner/model/loader.py
+++ b/src/llmtuner/model/loader.py
@@ -202,6 +202,7 @@ def load_model_and_tokenizer(
     if stage in ["rm", "ppo"]:
         model: "AutoModelForCausalLMWithValueHead" = AutoModelForCausalLMWithValueHead.from_pretrained(model)
         setattr(model, "_keys_to_ignore_on_save", [name for name, _ in model.named_parameters() if "pretrained_model" in name])
+        setattr(model, "tie_weights", MethodType(lambda _: None, model))
         vhead_path = (
             model_args.checkpoint_dir[-1] if model_args.checkpoint_dir is not None else model_args.model_name_or_path
         )